diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8c82bee16308e..24c15c4a3b6cd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -126,6 +126,7 @@ /.gitlab/deploy_dca/ @DataDog/container-integrations @DataDog/agent-delivery /.gitlab/deploy_packages/ @DataDog/agent-delivery +/.gitlab/deploy_packages/oci.yml @DataDog/agent-delivery @DataDog/fleet /.gitlab/deploy_packages/windows.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/deploy_packages/winget.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/deploy_packages/cluster_agent_cloudfoundry.yml @DataDog/platform-integrations @DataDog/agent-devx-infra @@ -148,7 +149,6 @@ /.gitlab/powershell_script_deploy @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/choco_build/choco_build.yml @DataDog/agent-delivery @DataDog/windows-agent -/.gitlab/choco_deploy/choco_deploy.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/integration_test/windows.yml @DataDog/agent-devx-infra @DataDog/windows-agent @@ -176,6 +176,7 @@ /.gitlab/package_build/ @DataDog/agent-delivery /.gitlab/package_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/package_build/installer.yml @DataDog/agent-delivery @DataDog/fleet /.gitlab/packaging/ @DataDog/agent-delivery /.gitlab/benchmarks/benchmarks.yml @DataDog/agent-apm @@ -271,6 +272,7 @@ /go.mod # do not notify anyone /go.sum # do not notify anyone + /Makefile.trace @DataDog/agent-apm /omnibus/ @DataDog/agent-delivery @@ -647,3 +649,16 @@ /internal/third_party/client-go @DataDog/container-platform /internal/third_party/kubernetes @DataDog/container-integrations /internal/third_party/golang/ @DataDog/container-integrations + +# With the introduction of go.work, dependencies bump modify go.mod and go.sum in a lot of file. +# Which bring a lot of team in the review each time. To make it smoother we no longer consider go.mod and go.sum owned by teams. +# Each team can individually decide to update CODEOWNERS to be requested for a review on each modification of their go.mod/sum +/**/go.mod # do not notify anyone +/**/go.sum # do not notify anyone + +# Add here modules that need explicit review from the team owning them +/internal/tools/**/go.mod @DataDog/agent-devx-loops +/internal/tools/**/go.sum @DataDog/agent-devx-loops + +/pkg/util/scrubber/go.mod @DataDog/agent-shared-components +/pkg/util/scrubber/go.sum @DataDog/agent-shared-components diff --git a/.github/workflows/add_milestone.yml b/.github/workflows/add_milestone.yml index 33885369f0e0c..6bcd28e967c2e 100644 --- a/.github/workflows/add_milestone.yml +++ b/.github/workflows/add_milestone.yml @@ -22,7 +22,7 @@ jobs: GH_REPO: ${{ github.repository }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml index 1138e8557d011..a480105bf17e8 100644 --- a/.github/workflows/buildimages-update.yml +++ b/.github/workflows/buildimages-update.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # credentials are needed to create the PR at the end of the workflow persist-credentials: true @@ -53,7 +53,7 @@ jobs: fi - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: ${{ steps.branch_fetch.outputs.RESULT == 'true' }} with: ref: ${{ inputs.branch }} diff --git a/.github/workflows/chase_release_managers.yml b/.github/workflows/chase_release_managers.yml index 436217659ae48..475068736efea 100644 --- a/.github/workflows/chase_release_managers.yml +++ b/.github/workflows/chase_release_managers.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/code_review_complexity.yml b/.github/workflows/code_review_complexity.yml index 8c26254f6c626..3f36b387e0f34 100644 --- a/.github/workflows/code_review_complexity.yml +++ b/.github/workflows/code_review_complexity.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bb6d39efa5e6d..bc52cb502bdaf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 4522ddd4100a2..66fbf9462a168 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -5,10 +5,12 @@ on: schedule: - cron: '0 14 * * 1,3,5' # Run on Monday, Wednesday, and Friday at 14:00 UTC - cron: '0 8 * * 1,3,5' # Same as above but at 08:00 UTC, to warn agent-integrations team about releasing + - cron: '0 9 * * 1' # Run Agent 6 workflow on Monday at 09:00 UTC env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + AGENT6_RELEASE_BRANCH: '6.53.x' + IS_AGENT6_RELEASE: ${{ github.event.schedule == '0 9 * * 1' }} permissions: {} jobs: @@ -19,18 +21,21 @@ jobs: warning: ${{ steps.warning.outputs.value }} steps: - name: Checkout repository + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: sparse-checkout: 'tasks' persist-credentials: false - name: Install python + if: ${{ env.IS_AGENT6_RELEASE == 'false' }} uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: 3.11 cache: "pip" - name: Install Python dependencies + if: ${{ env.IS_AGENT6_RELEASE == 'false' }} run: | python -m pip install --upgrade pip pip install -r requirements.txt @@ -40,7 +45,11 @@ jobs: - name: Determine the release active branches id: branches run: | - echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT + if ${{ env.IS_AGENT6_RELEASE == 'true' }}; then + echo "value=[\"$AGENT6_RELEASE_BRANCH\"]" >> $GITHUB_OUTPUT + else + echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT + fi - name: Set the warning option id: warning @@ -60,7 +69,7 @@ jobs: fail-fast: false steps: - name: Checkout the main branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: true @@ -93,11 +102,12 @@ jobs: fi - name: Create RC PR - if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}} + if: ${{ steps.check_for_changes.outputs.CHANGES == 'true' || env.IS_AGENT6_RELEASE == 'true' }} env: MATRIX: ${{ matrix.value }} run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git fetch - inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }} + if ${{ env.IS_AGENT6_RELEASE == 'true' }}; then + inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }} --patch-version + else + inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }} + fi diff --git a/.github/workflows/create_release_schedule.yml b/.github/workflows/create_release_schedule.yml index f8dc006160e3c..4b749ba3bdc97 100644 --- a/.github/workflows/create_release_schedule.yml +++ b/.github/workflows/create_release_schedule.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index b5f1ba4df7b0c..88f198f5b114d 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -20,7 +20,7 @@ permissions: {} jobs: generate: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -52,13 +52,13 @@ jobs: df -h - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} persist-credentials: false - name: Checkout btfhub-archive repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/btfhub-archive path: dev/dist/archive @@ -97,20 +97,20 @@ jobs: inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json --force-refresh - name: Upload artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: ./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json combine: needs: generate - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: contents: write pull-requests: write steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml index 3063c7c0db456..a77f4ba632c1d 100644 --- a/.github/workflows/datadog-static-analysis.yml +++ b/.github/workflows/datadog-static-analysis.yml @@ -11,7 +11,7 @@ jobs: name: Datadog Static Analyzer steps: - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Check code meets quality and security standards diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 8602adc372c1f..59e803598f6cd 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false # Fetch all history for applying timestamps to every page @@ -50,7 +50,7 @@ jobs: - name: Build documentation run: invoke docs.build - - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: documentation path: site diff --git a/.github/workflows/external-contributor.yml b/.github/workflows/external-contributor.yml index f0beb805c2a50..03cf46fd03511 100644 --- a/.github/workflows/external-contributor.yml +++ b/.github/workflows/external-contributor.yml @@ -17,7 +17,7 @@ jobs: if: github.event.pull_request.head.repo.full_name != github.repository steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: main fetch-depth: 0 diff --git a/.github/workflows/go-update-commenter.yml b/.github/workflows/go-update-commenter.yml index 1028110bc4fda..c74a404a09c4c 100644 --- a/.github/workflows/go-update-commenter.yml +++ b/.github/workflows/go-update-commenter.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: # get the Go version of the target branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -26,7 +26,7 @@ jobs: echo version="$(cat .go-version)" >> $GITHUB_OUTPUT # get the Go version of the PR branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Get current Go version diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml index e48d806c9dce9..4cc1a55690e6f 100644 --- a/.github/workflows/go_mod_tidy.yml +++ b/.github/workflows/go_mod_tidy.yml @@ -17,7 +17,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Checkout PR diff --git a/.github/workflows/gohai.yml b/.github/workflows/gohai.yml index 851c2f9d0ccd1..7ab3b5e49d2fc 100644 --- a/.github/workflows/gohai.yml +++ b/.github/workflows/gohai.yml @@ -34,7 +34,7 @@ jobs: go-file: [.go-version, pkg/gohai/go.mod] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 diff --git a/.github/workflows/label-analysis.yml b/.github/workflows/label-analysis.yml index ec53d5a695fed..7c4345e2ec323 100644 --- a/.github/workflows/label-analysis.yml +++ b/.github/workflows/label-analysis.yml @@ -23,7 +23,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false @@ -125,7 +125,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Setup python diff --git a/.github/workflows/markdown-lint-check.yml b/.github/workflows/markdown-lint-check.yml index 5f440614390ae..b5fe87f21f0f8 100644 --- a/.github/workflows/markdown-lint-check.yml +++ b/.github/workflows/markdown-lint-check.yml @@ -9,7 +9,7 @@ jobs: markdown-link-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: gaurav-nelson/github-action-markdown-link-check@d53a906aa6b22b8979d33bc86170567e619495ec # v1.0.15 diff --git a/.github/workflows/report-merged-pr.yml b/.github/workflows/report-merged-pr.yml index feefb3c5446bd..879b5b7e13330 100644 --- a/.github/workflows/report-merged-pr.yml +++ b/.github/workflows/report-merged-pr.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml index 195f3441c4591..a3a4ecbf609e5 100644 --- a/.github/workflows/serverless-benchmarks.yml +++ b/.github/workflows/serverless-benchmarks.yml @@ -24,7 +24,7 @@ jobs: sha: ${{ steps.prepare.outputs.sha }} steps: - name: Checkout ${{ github.base_ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -48,7 +48,7 @@ jobs: ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log - name: Upload result artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: baseline.log path: ${{runner.temp}}/benchmark.log @@ -63,7 +63,7 @@ jobs: steps: - name: Checkout ${{ github.ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.sha }} persist-credentials: false @@ -87,7 +87,7 @@ jobs: ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log - name: Upload result artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: current.log path: ${{runner.temp}}/benchmark.log diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml index 5bf7a69b36177..fda6772398b9e 100644 --- a/.github/workflows/serverless-binary-size.yml +++ b/.github/workflows/serverless-binary-size.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write # Add comment to PR steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false @@ -36,7 +36,7 @@ jobs: fi - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension @@ -152,7 +152,7 @@ jobs: done - name: Archive dependency graphs - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 if: steps.should.outputs.should_run == 'true' with: name: dependency-graphs diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml index f94ed5ca56b52..f11209b420ef0 100644 --- a/.github/workflows/serverless-integration.yml +++ b/.github/workflows/serverless-integration.yml @@ -26,13 +26,13 @@ jobs: name: ${{ matrix.suite }} on ${{ matrix.architecture }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false - name: Set up Node 20 - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: node-version: 20 @@ -40,7 +40,7 @@ jobs: run: sudo yarn global add serverless@^3.36.0 --prefix /usr/local - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension @@ -80,7 +80,7 @@ jobs: - name: Archive raw logs if: always() - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: rawlogs-${{ matrix.suite }}-${{ matrix.architecture }} path: ${{ steps.rawlogs.outputs.dir }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2761698a9b16..82599a4b84956 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,6 @@ include: - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml - - .gitlab/choco_deploy/choco_deploy.yml - .gitlab/powershell_script_signing/powershell_script_signing.yml - .gitlab/powershell_script_deploy/powershell_script_deploy.yml - .gitlab/common/shared.yml @@ -86,7 +85,7 @@ stages: - deploy_dca - choco_and_install_script_build - trigger_release - - choco_and_install_script_deploy + - install_script_deploy - internal_image_deploy - e2e_deploy - install_script_testing @@ -276,6 +275,7 @@ variables: # Feature flags FF_SCRIPT_SECTIONS: 1 # Prevent multiline scripts log collapsing, see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3392 FF_KUBERNETES_HONOR_ENTRYPOINT: true # Honor the entrypoint in the Docker image when running Kubernetes jobs + FF_TIMESTAMPS: true # # Condition mixins for simplification of rules @@ -623,18 +623,6 @@ workflow: - <<: *if_main_branch - <<: *if_release_branch -.on_main_or_release_branch_or_all_builds_or_pkg_installer_changes: - - <<: *if_main_branch - - <<: *if_release_branch - - <<: *if_run_all_builds - - <<: *if_run_all_unit_tests - - changes: - paths: - - pkg/fleet/**/* - compare_to: main - variables: - FAST_TESTS: "true" - .only_main: - <<: *if_not_main_branch when: never @@ -841,7 +829,9 @@ workflow: - !reference [.on_e2e_main_release_or_rc] - changes: paths: - # TODO: Add paths that should trigger tests for ASC + - cmd/**/* + - pkg/**/* + - comp/**/* - test/new-e2e/tests/agent-shared-components/**/* compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml index af5acfa72b74a..fd1af4a9868c5 100644 --- a/.gitlab/.ci-linters.yml +++ b/.gitlab/.ci-linters.yml @@ -37,6 +37,7 @@ needs-rules: - setup_agent_version - tests_ebpf_arm64 - tests_ebpf_x64 + - tests_macos_gitlab_amd64 - tests_windows_secagent_x64 - tests_windows_sysprobe_x64 - trigger_auto_staging_release @@ -56,8 +57,6 @@ job-owners: - build_otel_agent_binary_arm64 - build_otel_agent_binary_x64 - cancel-prev-pipelines - - check_pkg_size-amd64-a7 - - check_pkg_size-arm64-a7 - close_failing_tests_stale_issues - compute_gitlab_ci_config - deploy_cluster_agent_cloudfoundry @@ -83,7 +82,6 @@ job-owners: - new-e2e-unit-tests - ot_agent_deb-arm64-a7 - ot_agent_deb-x64-a7 - - publish_choco_7_x64 - publish_fakeintake - publish_fakeintake_latest - rc_kubernetes_deploy diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index e03f28a2d537d..2678bd7a5de6d 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -67,7 +67,7 @@ iot_agent_suse* @DataDog/agent-delivery dogstatsd_suse* @DataDog/agent-delivery agent_oci* @DataDog/agent-delivery installer_oci* @DataDog/agent-delivery -new_check_pkg_size @DataDog/agent-delivery +check_pkg_size @DataDog/agent-delivery # Testing package deploy deploy_deb_testing* @DataDog/agent-delivery diff --git a/.gitlab/choco_build/choco_build.yml b/.gitlab/choco_build/choco_build.yml index 8ec5bac814d18..06ff3c6bb5d0d 100644 --- a/.gitlab/choco_build/choco_build.yml +++ b/.gitlab/choco_build/choco_build.yml @@ -2,34 +2,8 @@ # choco_build stage # Contains jobs which build the chocolatey Agent package. -# Not currently used in the pipeline. -windows_choco_offline_7_x64: - rules: - !reference [.manual] - stage: choco_and_install_script_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_msi_and_bosh_zip_x64-a7"] - variables: - ARCH: "x64" - script: - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - copy omnibus\pkg\*.msi .\chocolatey\datadog-agent\offline\tools\ - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e AWS_NETWORKING=true - registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod offline -Flavor $FLAVOR -InstallDeps 1" - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - copy build-out\*.nupkg omnibus\pkg - artifacts: - expire_in: 2 weeks - paths: - - omnibus/pkg - -# The online version of the choco job gets the msi package through the gitlab artifacts -.windows_choco_online_7_x64: +# The choco job gets the msi package through the gitlab artifacts +.windows_choco_7_x64: rules: !reference [.on_deploy_stable_or_beta_repo_branch] stage: choco_and_install_script_build @@ -39,11 +13,7 @@ windows_choco_offline_7_x64: script: - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - $ErrorActionPreference = "Stop" - - mkdir temp\ - - copy omnibus\pkg\*.msi temp\ - - if (Test-Path omnibus) { remove-item -recurse -force omnibus } - if (Test-Path build-out) { remove-item -recurse -force build-out } - - mkdir omnibus\pkg - > docker run --rm -v "$(Get-Location):c:\mnt" @@ -52,10 +22,8 @@ windows_choco_offline_7_x64: -e BUCKET_BRANCH="$BUCKET_BRANCH" -e AWS_NETWORKING=true registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod online -MSIDirectory c:\mnt\temp -Flavor $FLAVOR -InstallDeps 1" + powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -MSIDirectory c:\mnt\omnibus\pkg -Flavor $FLAVOR -InstallDeps 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - Remove-Item -Path "temp\" -Recurse -Force - - copy build-out\*.nupkg omnibus\pkg - $CopyNupkgToS3 = "$S3_CP_CMD --recursive --exclude '*' --include '*.nupkg' build-out $S3_RELEASE_ARTIFACTS_URI/choco/nupkg" - Invoke-Expression $CopyNupkgToS3 artifacts: @@ -65,8 +33,8 @@ windows_choco_offline_7_x64: # Sometimes Chocolatey is flakey retry: 2 -windows_choco_online_7_x64: - extends: .windows_choco_online_7_x64 +windows_choco_7_x64: + extends: .windows_choco_7_x64 # On dev/PR branches: # - if the job is run manually it will create a package, but before the # package can be installed, the deploy_windows_testing-a7 job must @@ -75,8 +43,8 @@ windows_choco_online_7_x64: variables: FLAVOR: "datadog-agent" -windows_choco_online_7_x64-fips: - extends: .windows_choco_online_7_x64 +windows_choco_7_x64-fips: + extends: .windows_choco_7_x64 # On dev/PR branches: # - if the job is run manually it will create a package, but before the # package can be installed, the deploy_windows_testing-a7-fips job must diff --git a/.gitlab/choco_deploy/choco_deploy.yml b/.gitlab/choco_deploy/choco_deploy.yml deleted file mode 100644 index 05b0f0d83eba6..0000000000000 --- a/.gitlab/choco_deploy/choco_deploy.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# choco_build stage -# Contains a job which deploys the chocolatey Agent package. - -publish_choco_7_x64: - rules: !reference [.on_deploy_stable_or_beta_repo_branch_manual] - stage: choco_and_install_script_deploy - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_choco_online_7_x64"] - variables: - ARCH: "x64" - before_script: - - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile") - - If ($lastExitCode -ne "0") { exit "$lastExitCode" } - - $chocolateyApiKey=$(cat "$tmpfile") - - Remove-Item "$tmpfile" - script: - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - if (Test-Path nupkg) { remove-item -recurse -force nupkg } - - mkdir nupkg - - copy omnibus\pkg\*.nupkg nupkg\ - - Get-ChildItem nupkg - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e CHOCOLATEY_API_KEY=${chocolateyApiKey} - registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\chocopush.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index 7d435a4bc3a4b..944343f20da2f 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -1,7 +1,4 @@ --- variables: - # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev - # and check the job creating the image to make sure you have the right SHA prefix - TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" - # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 9c7c5005ca28 + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 221bbc806266 + TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: '' diff --git a/.gitlab/deploy_packages/oci.yml b/.gitlab/deploy_packages/oci.yml index bcefacac8d134..4d8ecadebfdff 100644 --- a/.gitlab/deploy_packages/oci.yml +++ b/.gitlab/deploy_packages/oci.yml @@ -28,6 +28,10 @@ include: - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${VERSION} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${CI_COMMIT_SHA} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} pipeline-${CI_PIPELINE_ID} + # necessary for ddstaging until we deprecate `-dev` packaging + - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} ${VERSION} + - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} ${CI_COMMIT_SHA} + - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} pipeline-${CI_PIPELINE_ID} variables: MAJOR_VERSION: 7 @@ -37,6 +41,7 @@ deploy_agent_oci: variables: OCI_PRODUCT: "datadog-agent" S3_PACKAGE: "agent-package" + S3_STAGING_PACKAGE: "agent-package-dev" deploy_installer_oci: extends: ".deploy_packages_oci" @@ -44,3 +49,4 @@ deploy_installer_oci: variables: OCI_PRODUCT: "datadog-installer" S3_PACKAGE: "installer-package" + S3_STAGING_PACKAGE: "installer-package-dev" diff --git a/.gitlab/dev_container_deploy/docker_linux.yml b/.gitlab/dev_container_deploy/docker_linux.yml index 5b61ffc403a2b..8994a6cbb9bce 100644 --- a/.gitlab/dev_container_deploy/docker_linux.yml +++ b/.gitlab/dev_container_deploy/docker_linux.yml @@ -80,6 +80,24 @@ dev_master-a7: - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64 IMG_DESTINATIONS: agent-dev:master-py3-jmx +dev_master-fips: + extends: .docker_publish_job_definition + stage: dev_container_deploy + rules: !reference [.on_main] + needs: + - docker_build_fips_agent7 + - docker_build_fips_agent7_arm64 + - docker_build_fips_agent7_jmx + - docker_build_fips_agent7_arm64_jmx + variables: + IMG_REGISTRIES: dev + parallel: + matrix: + - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-arm64 + IMG_DESTINATIONS: agent-dev:master-fips + - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-arm64 + IMG_DESTINATIONS: agent-dev:master-fips-jmx + dev_master-dogstatsd: extends: .docker_publish_job_definition stage: dev_container_deploy diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index b7f4b80cc6ed1..769ebe10138b8 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -216,26 +216,6 @@ kmt_run_secagent_tests_x64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_x64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - needs: - - kmt_setup_env_secagent_x64 - - upload_dependencies_secagent_x64 - - upload_secagent_tests_x64 - variables: - ARCH: "x86_64" - parallel: - matrix: - - TAG: - - "amazon_2023" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_x64_docker: extends: - .kmt_run_secagent_tests @@ -350,26 +330,6 @@ kmt_run_secagent_tests_arm64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_arm64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - needs: - - kmt_setup_env_secagent_arm64 - - upload_dependencies_secagent_arm64 - - upload_secagent_tests_arm64 - variables: - ARCH: "arm64" - parallel: - matrix: - - TAG: - - "ubuntu_24.04" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_arm64_docker: extends: - .kmt_run_secagent_tests @@ -426,7 +386,6 @@ kmt_secagent_tests_join_arm64: - kmt_run_secagent_tests_arm64 - kmt_run_secagent_tests_arm64_ad - kmt_run_secagent_tests_arm64_ebpfless - - kmt_run_secagent_tests_arm64_fentry - kmt_run_secagent_tests_arm64_docker kmt_secagent_cleanup_arm64: @@ -449,7 +408,6 @@ kmt_secagent_tests_join_x64: - kmt_run_secagent_tests_x64_required - kmt_run_secagent_tests_x64_ad - kmt_run_secagent_tests_x64_ebpfless - - kmt_run_secagent_tests_x64_fentry - kmt_run_secagent_tests_x64_docker kmt_secagent_cleanup_x64: diff --git a/.gitlab/notify/notify.yml b/.gitlab/notify/notify.yml index f40187465a50f..13131a7fecb4a 100644 --- a/.gitlab/notify/notify.yml +++ b/.gitlab/notify/notify.yml @@ -80,8 +80,8 @@ notify_github: script: - !reference [.setup_agent_github_app] # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is - # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 + # In particular, --break-system-packages command line option is + # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages - messagefile="$(mktemp)" - echo "Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:" >> "$messagefile" @@ -98,6 +98,7 @@ notify_gitlab_ci_changes: needs: [compute_gitlab_ci_config] tags: ["arch:amd64"] rules: + - !reference [.except_mergequeue] - changes: paths: - .gitlab-ci.yml @@ -105,8 +106,8 @@ notify_gitlab_ci_changes: compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 script: # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is - # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 + # In particular, --break-system-packages command line option is + # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages - !reference [.setup_agent_github_app] - inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml index ea458c9db77a7..88c93ac82c01c 100644 --- a/.gitlab/package_build/windows.yml +++ b/.gitlab/package_build/windows.yml @@ -36,7 +36,6 @@ -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} -e PIP_INDEX_URL=${PIP_INDEX_URL} -e API_KEY_ORG2=${API_KEY_ORG2} - -e OMNIBUS_GIT_CACHE_DIR=${Env:TEMP}/${CI_PIPELINE_ID}/omnibus-git-cache -e AGENT_FLAVOR=${AGENT_FLAVOR} registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index ceed774b62f41..ed09fa8ed1ee2 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -76,118 +76,14 @@ send_pkg_size: - inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 -.check_pkg_size: - stage: pkg_metrics - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - script: - - ls -l $OMNIBUS_PACKAGE_DIR - - if [[ "${ARCH}" == "amd64" ]]; then ls -l $OMNIBUS_PACKAGE_DIR_SUSE; fi - - - export failures=0 - - export last_stable=$(inv release.get-release-json-value "last_stable::${MAJOR_VERSION}" --no-worktree) - # Get stable packages from S3 buckets, send new package sizes & compare stable and new package sizes - # The loop assumes that all flavors start with "da", which is currently the case - # We want to run all package size comparisons before failing, so we set +e while doing the comparisons - # to get the error codes without exiting the shell. - - | - if [[ "${ARCH}" == "amd64" ]]; then ARCH_RPM_EXT="x86_64"; else ARCH_RPM_EXT="aarch64"; fi - for flavor in ${FLAVORS}; do - - if [[ "${ARCH}" == "amd64" && "$flavor" != "datadog-heroku-agent" ]]; then - mkdir -p "/tmp/stable/${flavor}/suse" - curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/suse/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - set +e - inv package.compare-size --package-type "${flavor} suse rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR_SUSE/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - failures=$((${failures}+$?)) - set -e - fi - - mkdir -p "/tmp/stable/${flavor}" - - curl -sSL "https://s3.amazonaws.com/apt.datadoghq.com/pool/d/da/${flavor}_${last_stable}-1_${ARCH}.deb" -o "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb" - - set +e - inv package.compare-size --package-type "${flavor} deb" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}_${MAJOR_VERSION}*_${ARCH}.deb" --stable-package "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb" - failures=$((${failures}+$?)) - set -e - - if [[ "$flavor" != "datadog-heroku-agent" && ( "${ARCH}" == "amd64" || "$flavor" != "datadog-dogstatsd") ]]; then - # We don't build RPM packages for the heroku flavor - curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - set +e - inv package.compare-size --package-type "${flavor} rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - failures=$((${failures}+$?)) - set -e - fi - done - - # Make the job fail if at least one package is above threshold - - if [ "${failures}" -ne "0" ]; then false; fi - -check_pkg_size-amd64-a7: - extends: .check_pkg_size - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: - - agent_deb-x64-a7 - - iot_agent_deb-x64 - - dogstatsd_deb-x64 - - agent_heroku_deb-x64-a7 - - agent_rpm-x64-a7 - - iot_agent_rpm-x64 - - dogstatsd_rpm-x64 - - agent_suse-x64-a7 - - dogstatsd_suse-x64 - - iot_agent_suse-x64 - variables: - MAJOR_VERSION: 7 - FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd datadog-heroku-agent" - ARCH: "amd64" - before_script: - # FIXME: ["datadog-agent"]="140000000" and ["datadog-heroku-agent"]="140000000" should - # be replaced by "50000000" - # "70000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="140000000" - ["datadog-iot-agent"]="10000000" - ["datadog-dogstatsd"]="10000000" - ["datadog-heroku-agent"]="70000000" - ) - -check_pkg_size-arm64-a7: - extends: .check_pkg_size - rules: !reference [.on_all_builds] - needs: - - agent_deb-arm64-a7 - - iot_agent_deb-arm64 - - dogstatsd_deb-arm64 - - agent_rpm-arm64-a7 - - iot_agent_rpm-arm64 - variables: - MAJOR_VERSION: 7 - FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd" - ARCH: "arm64" - before_script: - # FIXME: ["datadog-agent"]="140000000" should be replaced by "70000000" - # "140000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="140000000" - ["datadog-iot-agent"]="10000000" - ["datadog-dogstatsd"]="10000000" - ) -new_check_pkg_size: +check_pkg_size: stage: pkg_metrics image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] rules: - !reference [.except_mergequeue] - when: on_success - allow_failure: true needs: - agent_deb-x64-a7 - iot_agent_deb-x64 diff --git a/.gitlab/powershell_script_deploy/powershell_script_deploy.yml b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml index 4760dfdf18030..5269045adde05 100644 --- a/.gitlab/powershell_script_deploy/powershell_script_deploy.yml +++ b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml @@ -5,7 +5,7 @@ powershell_script_deploy: image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - stage: choco_and_install_script_deploy + stage: install_script_deploy rules: !reference [.manual] needs: ["powershell_script_signing"] @@ -19,7 +19,7 @@ powershell_script_deploy: windows_bootstrapper_deploy: image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - stage: choco_and_install_script_deploy + stage: install_script_deploy rules: !reference [.manual] needs: ["windows-installer-amd64"] diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index da5ed4d68c8cf..1a92a354d47b8 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -2,44 +2,8 @@ include: - .gitlab/common/macos.yml -tests_macos: - stage: source_test - # HACK: Run macOS unit tests only on full pipelines, to limit the use of macOS GitHub runners. - rules: - - !reference [.on_main_or_release_branch_or_all_builds_or_pkg_installer_changes] - - when: manual - allow_failure: true - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["setup_agent_version"] - script: - - !reference [.setup_macos_github_app] - - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - - export VERSION_CACHE_CONTENT=$(cat agent-version.cache | base64 -) - # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is - # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages - - FAST_TESTS_FLAG="" - - if [[ "$FAST_TESTS" = "true" ]]; then FAST_TESTS_FLAG="--fast-tests true"; fi - - inv -e github.trigger-macos --workflow-type "test" --datadog-agent-ref "$CI_COMMIT_SHA" --version-cache "$VERSION_CACHE_CONTENT" $FAST_TESTS_FLAG --test-washer - timeout: 6h - after_script: - - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "junit-*-repacked.tgz" - artifacts: - expire_in: 2 weeks - when: always - paths: - - test_output.json - - junit-*-repacked.tgz - reports: - junit: "**/junit-out-*.xml" - .tests_macos_gitlab: stage: source_test - rules: - - !reference [.except_mergequeue] - - when: on_success extends: .macos_gitlab needs: ["go_deps", "go_tools_deps"] variables: diff --git a/CHANGELOG-DCA.rst b/CHANGELOG-DCA.rst index 5ed8fdc3430f3..7b5abd00a3bc2 100644 --- a/CHANGELOG-DCA.rst +++ b/CHANGELOG-DCA.rst @@ -2,6 +2,46 @@ Release Notes ============= +.. _Release Notes_7.60.1: + +7.60.1 +====== + +.. _Release Notes_7.60.1_Prelude: + +Prelude +------- + +Released on: 2024-12-19 +Pinned to datadog-agent v7.60.1: `CHANGELOG `_. + +.. _Release Notes_7.60.0: + +7.60.0 +====== + +.. _Release Notes_7.60.0_Prelude: + +Prelude +------- + +Released on: 2024-12-16 +Pinned to datadog-agent v7.60.0: `CHANGELOG `_. + + +.. _Release Notes_7.60.0_Bug Fixes: + +Bug Fixes +--------- + +- Fixes bug where incorrect timestamp would be used for unbundled Kubernetes events. + +- Fixed an issue in the KSM check when it's configured with the option + ``pod_collection_mode`` set to ``node_kubelet``. Previously, the check could + fail to start if there was a timeout while contacting the API server. This + issue has now been resolved. + + .. _Release Notes_7.59.1: 7.59.1 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9dcb70ac5e1e1..1def73a37b2e1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,159 @@ Release Notes ============= +.. _Release Notes_7.60.1: + +7.60.1 +====== + +.. _Release Notes_7.60.1_Prelude: + +Prelude +------- + +Release on: 2024-12-19 + + +.. _Release Notes_7.60.1_Security Notes: + +Security Notes +-------------- + +- Update ``golang.org/x/crypto`` to fix CVE-2024-45337. + + +.. _Release Notes_7.60.0: + +7.60.0 +====== + +.. _Release Notes_7.60.0_Prelude: + +Prelude +------- + +Release on: 2024-12-16 + +- Please refer to the `7.60.0 tag on integrations-core `_ for the list of changes on the Core Checks + + +.. _Release Notes_7.60.0_Upgrade Notes: + +Upgrade Notes +------------- + +- * Parameter ``peer_tags_aggregation`` (a.k.a. environment variable ``DD_APM_PEER_TAGS_AGGREGATION``) is now enabled by default. This means that aggregation of peer related tags (e.g., `peer.service`, `db.instance`, etc.) now happens in the Agent, which enables statistics for Inferred Entities. If you want to disable this feature, set `peer_tags_aggregation` to `false` in your Agent configuration. + + * Parameter ``compute_stats_by_span_kind`` (a.k.a. environment variable ``DD_APM_COMPUTE_STATS_BY_SPAN_KIND``) is now enabled by default. This means spans with an eligible `span.kind` will have stats computed. If disabled, only top-level and measured spans will have stats computed. If you want to disable this feature, set `compute_stats_by_span_kind` to `false` in your Agent configuration. + + Note: When using ``peer_tags_aggregation`` and ``compute_stats_by_span_kind``, a high cardinality of peer tags or APM resources can contribute to higher CPU and memory consumption. If enabling both causes the Agent to consume too many resources, try disabling `compute_stats_by_span_kind` first. + + It is recommended that you update your tracing libraries according to the instructions `here `_ and set ``DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED`` (or ``dd.trace.remove.integration-service-names.enabled``) to ``true``. + +- Upgraded JMXFetch to `0.49.5 `_ which adds support for ``UnloadedClassCount`` metric + and IBM J9 gc metrics. See `0.49.5 `_ for more details. + + +.. _Release Notes_7.60.0_New Features: + +New Features +------------ + +- `Inferred Service dependencies `_ are now Generally Available (exiting Beta) and enabled by default. Inferred Services of all kinds now have trace metrics and are available in dependency maps. `apm_config.peer_tags_aggregation` and `apm_config.compute_stats_by_span_kind` both now default to `true` unless explicitly set to `false`. + +- Add `check_tag_cardinality` parameter config check. + + By default `check_tag_cardinality` is not set which doesn't change the behavior of the checks. + Once it is set in pod annotaions, it overrides the cardinality value provided in the base agent configuration. + Example of usage: + ```yaml + ad.datadoghq.com/redis.checks: | + { + "redisdb": { + "check_tag_cardinality": "high", + "instances": [ + { + "host": "%%host%%", + "port": "6379" + } + ] + } + } + ``` + +- Added a new feature flag `enable_receive_resource_spans_v2` in DD_APM_FEATURES that gates a refactored implementation of ReceiveResourceSpans for OTLP. + + +.. _Release Notes_7.60.0_Enhancement Notes: + +Enhancement Notes +----------------- + +- Added information about where the Agent sourced BTF data for eBPF to the Agent flare. When applicable, this will appear in ``system-probe/ebpf_btf_loader.log``. + +- The Agent flare now returns NAT debug information from conntrack in the ``system-probe`` directory. + +- The ``flare`` subcommand includes a ``--provider-timeout`` option to set a timeout for each file collection (default is 10s), useful for unblocking slow flare creation. + +- This change reduces the number of DNS queries made by Network Traffic + based paths in Network Path. + A cache of reverse DNS lookups is used to reduce the number of DNS + queries. Additionally, reverse DNS lookups are now performed only + for private IPs and not for public IPs. + +- Agent flare now includes system-probe telemetry data via ``system-probe/system_probe_telemetry.log``. + +- The MSI installer uses 7zr.exe to decompress the embedded Python. + +- On Windows, the endpoint /windows_crash_detection/check has been modified to report crashes in + an asynchronous manner, to allow processing of large crash dumps without blocking or timing out. + The first check will return a busy status and continue to do so until the processing is completed. + + +.. _Release Notes_7.60.0_Deprecation Notes: + +Deprecation Notes +----------------- + +- Prebuilt eBPF for the network tracer system-probe module has been + deprecated in favor of CO-RE and runtime compilation variants on Linux + kernel versions 6+ and RHEL kernel versions 5.14+. To continue to use + the prebuilt eBPF network tracer, set + `system_probe_config.allow_prebuilt_fallback` in the + system-probe config file, or set the environment variable + `DD_ALLOW_PREBUILT_FALLBACK`, to `true` on these platforms. + +- The feature flag `service_monitoring_config.enable_http_stats_by_status_code` was deprecated and removed. + No impact on USM's behavior. + + +.. _Release Notes_7.60.0_Bug Fixes: + +Bug Fixes +--------- + +- Fixes an issue added in 7.50 that causes the Windows event log tailer to drop + events if it cannot open their publisher metadata. + +- Fix a bug in the config parser that broke ignored_ip_addresses from working in NDM Autodiscovery. + +- Fixes host tags with a configurable duration so the metric's context hash doesn't change, preventing the aggregator from mistaking it as a new metric. + +- Fix `could not parse voltage fields` error in Nvidia Jetson integration when tegrastats output contains mW units. + +- Fix building of Python extension containing native code. + +- [oracle] Fix broken activity sampling with an external Oracle client. + +- Fix nil pointer error on Oracle DBM query when the check's connection is lost before SELECT statement executes. + +- Fix a regression that caused the Agent to not be able to run if its + capabilities had been modified with the `setcap` command. + +- Fix bug wherein single line truncated logs ended with whitespace characters were not being tagged as truncated. + Fix issue with the truncation message occasionally causing subsequent logs to think they were truncated when they were not (single line logs only). + + .. _Release Notes_7.59.1: 7.59.1 diff --git a/Dockerfiles/agent-ot/Dockerfile.agent-otel b/Dockerfiles/agent-ot/Dockerfile.agent-otel index 78534598a581c..49ea309af7019 100644 --- a/Dockerfiles/agent-ot/Dockerfile.agent-otel +++ b/Dockerfiles/agent-ot/Dockerfile.agent-otel @@ -1,5 +1,5 @@ -ARG AGENT_VERSION=7.57.0-v1.0-ot-beta-jmx -ARG AGENT_BRANCH=7.57.x-otel-beta-v1 +ARG AGENT_VERSION=7.59.0-v1.1.0-ot-beta-jmx +ARG AGENT_BRANCH=7.59.x # Use the Ubuntu Slim AMD64 base image FROM ubuntu:24.04 AS builder diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index d424a9ed015c7..13955c7e8a940 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -1209,8 +1209,6 @@ core,github.com/google/gopacket/afpacket,BSD-3-Clause,"Copyright (c) 2009-2011 A core,github.com/google/gopacket/layers,BSD-3-Clause,"Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. | Copyright (c) 2012 Google, Inc. All rights reserved." core,github.com/google/gopacket/pcap,BSD-3-Clause,"Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. | Copyright (c) 2012 Google, Inc. All rights reserved." core,github.com/google/gopacket/pcapgo,BSD-3-Clause,"Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. | Copyright (c) 2012 Google, Inc. All rights reserved." -core,github.com/google/licenseclassifier/v2,Apache-2.0,Copyright 2017 Google LLC All Rights Reserved. | Copyright 2020 Google LLC All Rights Reserved. -core,github.com/google/licenseclassifier/v2/assets,Apache-2.0,Copyright 2017 Google LLC All Rights Reserved. | Copyright 2020 Google LLC All Rights Reserved. core,github.com/google/pprof/profile,Apache-2.0,Andrew Hunter | Google Inc. | Hyoun Kyu Cho | Martin Spier | Raul Silvera | Taco de Wolff | Tipp Moseley core,github.com/google/s2a-go,Apache-2.0,Copyright (c) 2020 Google core,github.com/google/s2a-go/fallback,Apache-2.0,Copyright (c) 2020 Google @@ -2043,15 +2041,6 @@ core,github.com/secure-systems-lab/go-securesystemslib/dsse,MIT,Copyright (c) 20 core,github.com/secure-systems-lab/go-securesystemslib/signerverifier,MIT,Copyright (c) 2021 NYU Secure Systems Lab core,github.com/sergi/go-diff/diffmatchpatch,MIT,Copyright (c) 2012-2016 The go-diff Authors. All rights reserved | Danny Yoo | James Kolb | Jonathan Amsterdam | Markus Zimmermann | Matt Kovars | Osman Masood | Robert Carlsen | Rory Flynn | Sergi Mansilla | Shatrugna Sadhu | Shawn Smith | Stas Maksimov | Tor Arvid Lund | Zac Bergquist | Örjan Persson core,github.com/shibumi/go-pathspec,Apache-2.0,Copyright (c) 2012 The Go Authors. All rights reserved. -core,github.com/shirou/gopsutil/v3/common,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/cpu,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/disk,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/host,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/internal/common,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/load,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/mem,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/net,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" -core,github.com/shirou/gopsutil/v3/process,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" core,github.com/shirou/gopsutil/v4/common,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" core,github.com/shirou/gopsutil/v4/cpu,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" core,github.com/shirou/gopsutil/v4/disk,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" @@ -2062,7 +2051,6 @@ core,github.com/shirou/gopsutil/v4/mem,BSD-3-Clause,"Copyright (c) 2009 The Go A core,github.com/shirou/gopsutil/v4/net,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" core,github.com/shirou/gopsutil/v4/process,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright (c) 2014, WAKAYAMA Shirou" core,github.com/shirou/w32,BSD-3-Clause,Copyright (c) 2010-2012 The w32 Authors. All rights reserved. -core,github.com/shoenig/go-m1cpu,MPL-2.0,"copyright doctrines of fair use, fair dealing, or other" core,github.com/shopspring/decimal,MIT,"Copyright (c) 2013 Oguz Bilgic | Copyright (c) 2015 Spring, Inc" core,github.com/signalfx/sapm-proto/client,Apache-2.0,"Copyright 2019 Splunk, Inc." core,github.com/signalfx/sapm-proto/gen,Apache-2.0,"Copyright 2019 Splunk, Inc." @@ -2918,9 +2906,6 @@ core,google.golang.org/protobuf/types/known/timestamppb,BSD-3-Clause,Copyright ( core,google.golang.org/protobuf/types/known/wrapperspb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,google.golang.org/protobuf/types/pluginpb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,gopkg.in/DataDog/dd-trace-go.v1/appsec/events,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/datastreams/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext,Apache-2.0,"Copyright 2016-Present Datadog, Inc." diff --git a/chocolatey/datadog-agent/online/datadog-agent-online.nuspec b/chocolatey/datadog-agent/datadog-agent.nuspec similarity index 97% rename from chocolatey/datadog-agent/online/datadog-agent-online.nuspec rename to chocolatey/datadog-agent/datadog-agent.nuspec index 73f96bfb83871..f3d9ec86ca694 100644 --- a/chocolatey/datadog-agent/online/datadog-agent-online.nuspec +++ b/chocolatey/datadog-agent/datadog-agent.nuspec @@ -1,4 +1,4 @@ - + datadog-agent diff --git a/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec b/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec deleted file mode 100644 index aedf9555a1c51..0000000000000 --- a/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec +++ /dev/null @@ -1,31 +0,0 @@ - - - - datadog-agent-offline - $package_version$ - https://github.com/DataDog/datadog-agent/tree/main/chocolatey - Datadog - Datadog Agent Offline Install - Datadog - https://github.com/DataDog/datadog-agent - https://datadog-prod.imgix.net/img/dd_logo_70x75.png - $copyright$ - https://raw.githubusercontent.com/DataDog/datadog-agent/main/LICENSE - true - https://docs.datadoghq.com - datadog agent monitoring admin - The Datadog Agent for Microsoft Windows - The Datadog Agent faithfully collects events and metrics and brings them to Datadog on your behalf so that you can do something useful with your monitoring and performance data. - -## Package settings - -You may set [custom settings](https://docs.datadoghq.com/agent/basic_agent_usage/windows/?tab=commandline#installation) to the Agent when installing by using the [`--installer-arguments` option of `choco install`](https://chocolatey.org/docs/getting-started#overriding-default-install-directory-or-other-advanced-install-concepts). - -For example, to set the API key you may run: -`choco install -ia="APIKEY=""YOUR_DATADOG_API_KEY""" datadog-agent-offline` - $release_notes$ - - - - - diff --git a/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 b/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 deleted file mode 100644 index 98d39c92654e0..0000000000000 --- a/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -$ErrorActionPreference = 'Stop'; - -$toolsDir = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)" -$nupkgs = Get-ChildItem $toolsDir\datadog-agent*.msi -if (($nupkgs | Measure-Object).Count -gt 1) { - Write-Host "More than 1 MSI installer exists - aborting" - exit -2 -} -$packageArgs = @{ - packageName = $env:ChocolateyPackageName - unzipLocation = $toolsDir - fileType = 'msi' - file = $nupkgs[0].FullName - softwareName = 'Datadog Agent' - silentArgs = "/qn /norestart /l*v `"$($env:TEMP)\$($packageName).$($env:chocolateyPackageVersion).MsiInstall.log`"" - validExitCodes= @(0, 3010, 1641) -} -Install-ChocolateyInstallPackage @packageArgs - -$installInfo = @" ---- -install_method: - tool: chocolatey - tool_version: chocolatey-$($env:CHOCOLATEY_VERSION) - installer_version: chocolatey_package-offline -"@ - -$appDataDir = (Get-ItemProperty -Path "HKLM:\SOFTWARE\Datadog\Datadog Agent").ConfigRoot -Out-File -FilePath $appDataDir\install_info -InputObject $installInfo diff --git a/chocolatey/datadog-agent/online/tools/VERIFICATION.txt b/chocolatey/datadog-agent/online/tools/VERIFICATION.txt deleted file mode 100644 index 5ef7f466e2cd6..0000000000000 --- a/chocolatey/datadog-agent/online/tools/VERIFICATION.txt +++ /dev/null @@ -1,5 +0,0 @@ -VERIFICATION -Verification is intended to assist the Chocolatey moderators and community in verifying that this package's contents are trustworthy. - -This package is published by Datadog itself. -The binaries are identical to other package types for the Datadog Agent. diff --git a/chocolatey/datadog-agent/offline/tools/VERIFICATION.txt b/chocolatey/datadog-agent/tools/VERIFICATION.txt similarity index 100% rename from chocolatey/datadog-agent/offline/tools/VERIFICATION.txt rename to chocolatey/datadog-agent/tools/VERIFICATION.txt diff --git a/chocolatey/datadog-agent/online/tools/chocolateyinstall.ps1 b/chocolatey/datadog-agent/tools/chocolateyinstall.ps1 similarity index 100% rename from chocolatey/datadog-agent/online/tools/chocolateyinstall.ps1 rename to chocolatey/datadog-agent/tools/chocolateyinstall.ps1 diff --git a/chocolatey/datadog-fips-agent/online/datadog-fips-agent-online.nuspec b/chocolatey/datadog-fips-agent/datadog-fips-agent.nuspec similarity index 100% rename from chocolatey/datadog-fips-agent/online/datadog-fips-agent-online.nuspec rename to chocolatey/datadog-fips-agent/datadog-fips-agent.nuspec diff --git a/chocolatey/datadog-fips-agent/online/tools/VERIFICATION.txt b/chocolatey/datadog-fips-agent/tools/VERIFICATION.txt similarity index 100% rename from chocolatey/datadog-fips-agent/online/tools/VERIFICATION.txt rename to chocolatey/datadog-fips-agent/tools/VERIFICATION.txt diff --git a/chocolatey/datadog-fips-agent/online/tools/chocolateyinstall.ps1 b/chocolatey/datadog-fips-agent/tools/chocolateyinstall.ps1 similarity index 100% rename from chocolatey/datadog-fips-agent/online/tools/chocolateyinstall.ps1 rename to chocolatey/datadog-fips-agent/tools/chocolateyinstall.ps1 diff --git a/cmd/agent/common/autodiscovery.go b/cmd/agent/common/autodiscovery.go index 8ff855aeca19f..67a220b706750 100644 --- a/cmd/agent/common/autodiscovery.go +++ b/cmd/agent/common/autodiscovery.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "time" "go.uber.org/atomic" @@ -41,6 +42,10 @@ var ( ) func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, ac autodiscovery.Component) { + if pkgconfigsetup.Datadog().GetString("fleet_policies_dir") != "" { + confSearchPaths = append(confSearchPaths, filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d")) + } + providers.InitConfigFilesReader(confSearchPaths) acTelemetryStore := ac.GetTelemetryStore() diff --git a/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default b/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default index 00d9a2dbba2c8..c338654dc4e71 100644 --- a/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default +++ b/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default @@ -1,2 +1,15 @@ +## This file is overwritten upon Agent upgrade. +## To make modifications to the check configuration, copy this file +## to `cpu.yaml` and make your changes on that file. + +init_config: + +## The Cpu check only supports one configured instance. + instances: - - {} + - + + ## @param report_total_percpu - boolean - required + ## Set to true to report cpu total metrics for each cpu. + # + # report_total_percpu: false diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index 1b4fb2d3bc1e3..a6da5391ae5d5 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -175,10 +175,18 @@ func readProfileData(seconds int) (flare.ProfileData, error) { type pprofGetter func(path string) ([]byte, error) - tcpGet := func(portConfig string) pprofGetter { - pprofURL := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof", pkgconfigsetup.Datadog().GetInt(portConfig)) + tcpGet := func(portConfig string, onHTTPS bool) pprofGetter { + endpoint := url.URL{ + Scheme: "http", + Host: net.JoinHostPort("127.0.0.1", strconv.Itoa(pkgconfigsetup.Datadog().GetInt(portConfig))), + Path: "/debug/pprof", + } + if onHTTPS { + endpoint.Scheme = "https" + } + return func(path string) ([]byte, error) { - return util.DoGet(c, pprofURL+path, util.LeaveConnectionOpen) + return util.DoGet(c, endpoint.String()+path, util.LeaveConnectionOpen) } } @@ -228,15 +236,15 @@ func readProfileData(seconds int) (flare.ProfileData, error) { } agentCollectors := map[string]agentProfileCollector{ - "core": serviceProfileCollector(tcpGet("expvar_port"), seconds), - "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port"), seconds), + "core": serviceProfileCollector(tcpGet("expvar_port", false), seconds), + "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port", false), seconds), } if pkgconfigsetup.Datadog().GetBool("process_config.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.container_collection.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.process_collection.enabled") { - agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port"), seconds) + agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port", false), seconds) } if pkgconfigsetup.Datadog().GetBool("apm_config.enabled") { @@ -249,7 +257,7 @@ func readProfileData(seconds int) (flare.ProfileData, error) { traceCpusec = 4 } - agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port"), traceCpusec) + agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port", true), traceCpusec) } if pkgconfigsetup.SystemProbe().GetBool("system_probe_config.enabled") { diff --git a/cmd/agent/subcommands/flare/command_test.go b/cmd/agent/subcommands/flare/command_test.go index a27304e95b133..4c96cae079aa9 100644 --- a/cmd/agent/subcommands/flare/command_test.go +++ b/cmd/agent/subcommands/flare/command_test.go @@ -29,6 +29,7 @@ type commandTestSuite struct { suite.Suite sysprobeSocketPath string tcpServer *httptest.Server + tcpTLSServer *httptest.Server unixServer *httptest.Server systemProbeServer *httptest.Server } @@ -42,13 +43,17 @@ func (c *commandTestSuite) SetupSuite() { // This should be called by each test that requires them. func (c *commandTestSuite) startTestServers() { t := c.T() - c.tcpServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() + c.tcpServer, c.tcpTLSServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() t.Cleanup(func() { if c.tcpServer != nil { c.tcpServer.Close() c.tcpServer = nil } + if c.tcpTLSServer != nil { + c.tcpTLSServer.Close() + c.tcpTLSServer = nil + } if c.unixServer != nil { c.unixServer.Close() c.unixServer = nil @@ -82,12 +87,13 @@ func newMockHandler() http.HandlerFunc { }) } -func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { +func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, tcpTLSServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { var err error t := c.T() handler := newMockHandler() tcpServer = httptest.NewServer(handler) + tcpTLSServer = httptest.NewTLSServer(handler) if runtime.GOOS == "linux" { unixServer = httptest.NewUnstartedServer(handler) unixServer.Listener, err = net.Listen("unix", c.sysprobeSocketPath) @@ -101,7 +107,7 @@ func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, uni sysProbeServer.Start() } - return tcpServer, unixServer, sysProbeServer + return tcpServer, tcpTLSServer, unixServer, sysProbeServer } func TestCommandTestSuite(t *testing.T) { @@ -116,10 +122,14 @@ func (c *commandTestSuite) TestReadProfileData() { require.NoError(t, err) port := u.Port() + u, err = url.Parse(c.tcpTLSServer.URL) + require.NoError(t, err) + httpsPort := u.Port() + mockConfig := configmock.New(t) mockConfig.SetWithoutSource("expvar_port", port) mockConfig.SetWithoutSource("apm_config.enabled", true) - mockConfig.SetWithoutSource("apm_config.debug.port", port) + mockConfig.SetWithoutSource("apm_config.debug.port", httpsPort) mockConfig.SetWithoutSource("apm_config.receiver_timeout", "10") mockConfig.SetWithoutSource("process_config.expvar_port", port) mockConfig.SetWithoutSource("security_agent.expvar_port", port) diff --git a/cmd/agent/subcommands/secret/command.go b/cmd/agent/subcommands/secret/command.go index e24f95ca6d693..d8b2a7048114f 100644 --- a/cmd/agent/subcommands/secret/command.go +++ b/cmd/agent/subcommands/secret/command.go @@ -101,7 +101,7 @@ func traceAgentSecretRefresh(conf config.Component) ([]byte, error) { c := apiutil.GetClient(false) c.Timeout = conf.GetDuration("server_timeout") * time.Second - url := fmt.Sprintf("http://127.0.0.1:%d/secret/refresh", port) + url := fmt.Sprintf("https://127.0.0.1:%d/secret/refresh", port) res, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return nil, fmt.Errorf("could not contact trace-agent: %s", err) diff --git a/cmd/cluster-agent/api/server.go b/cmd/cluster-agent/api/server.go index 37efad4d900dd..c4f5c0534633a 100644 --- a/cmd/cluster-agent/api/server.go +++ b/cmd/cluster-agent/api/server.go @@ -12,6 +12,7 @@ package api import ( "context" + "crypto/subtle" "crypto/tls" "crypto/x509" "encoding/pem" @@ -116,7 +117,7 @@ func StartServer(ctx context.Context, w workloadmeta.Component, taggerComp tagge logWriter, _ := pkglogsetup.NewTLSHandshakeErrorWriter(4, log.WarnLvl) authInterceptor := grpcutil.AuthInterceptor(func(token string) (interface{}, error) { - if token != util.GetDCAAuthToken() { + if subtle.ConstantTimeCompare([]byte(token), []byte(util.GetDCAAuthToken())) == 0 { return struct{}{}, errors.New("Invalid session token") } diff --git a/cmd/installer-downloader/main.go b/cmd/installer-downloader/main.go index f7b67910ae94e..60e2141af944e 100644 --- a/cmd/installer-downloader/main.go +++ b/cmd/installer-downloader/main.go @@ -44,12 +44,12 @@ func main() { ctx := context.Background() t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor)) - _ = t.Start(ctx) - defer func() { _ = t.Stop(ctx) }() var err error span, ctx := telemetry.StartSpanFromEnv(ctx, fmt.Sprintf("downloader-%s", Flavor)) - defer func() { span.Finish(err) }() err = runDownloader(ctx, env, Version, Flavor) + + span.Finish(err) + t.Stop() if err != nil { fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err) os.Exit(1) diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 66ec6b1c74c57..f394d9955d407 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -87,7 +87,7 @@ func UnprivilegedCommands(_ *command.GlobalParams) []*cobra.Command { type cmd struct { t *telemetry.Telemetry ctx context.Context - span telemetry.Span + span *telemetry.Span env *env.Env } @@ -107,10 +107,7 @@ func newCmd(operation string) *cmd { func (c *cmd) Stop(err error) { c.span.Finish(err) if c.t != nil { - err := c.t.Stop(context.Background()) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to stop telemetry: %v\n", err) - } + c.t.Stop() } } @@ -225,11 +222,6 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { site = config.Site } t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands - err := t.Start(context.Background()) - if err != nil { - fmt.Printf("failed to start telemetry: %v\n", err) - return nil - } return t } diff --git a/cmd/installer/subcommands/installer/umask_nix.go b/cmd/installer/subcommands/installer/umask_nix.go index 1fd44c01ac405..dec64eccd6860 100644 --- a/cmd/installer/subcommands/installer/umask_nix.go +++ b/cmd/installer/subcommands/installer/umask_nix.go @@ -14,7 +14,7 @@ import ( ) // setInstallerUmask sets umask 0 to override any inherited umask -func setInstallerUmask(span telemetry.Span) { +func setInstallerUmask(span *telemetry.Span) { oldmask := syscall.Umask(0) span.SetTag("inherited_umask", oldmask) } diff --git a/cmd/installer/subcommands/installer/umask_windows.go b/cmd/installer/subcommands/installer/umask_windows.go index 1b076f92bc389..d8661700cd56e 100644 --- a/cmd/installer/subcommands/installer/umask_windows.go +++ b/cmd/installer/subcommands/installer/umask_windows.go @@ -10,4 +10,4 @@ package installer import "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" // setInstallerUmask no-op on Windows -func setInstallerUmask(_ telemetry.Span) {} +func setInstallerUmask(_ *telemetry.Span) {} diff --git a/cmd/otel-agent/config/agent_config_test.go b/cmd/otel-agent/config/agent_config_test.go index 2d08cfb4ac58f..01567d48d7ffe 100644 --- a/cmd/otel-agent/config/agent_config_test.go +++ b/cmd/otel-agent/config/agent_config_test.go @@ -78,7 +78,7 @@ func (suite *ConfigTestSuite) TestAgentConfigDefaults() { assert.Equal(t, 6, c.Get("logs_config.compression_level")) assert.Equal(t, "https://trace.agent.datadoghq.com", c.Get("apm_config.apm_dd_url")) assert.Equal(t, false, c.Get("apm_config.receiver_enabled")) - assert.Equal(t, true, c.Get("otlp_config.traces.span_name_as_resource_name")) + assert.Equal(t, false, c.Get("otlp_config.traces.span_name_as_resource_name")) assert.Equal(t, []string{"enable_receive_resource_spans_v2", "enable_operation_and_resource_name_logic_v2", "enable_otlp_compute_top_level_by_span_kind"}, c.Get("apm_config.features")) } @@ -104,7 +104,7 @@ func (suite *ConfigTestSuite) TestAgentConfigWithDatadogYamlDefaults() { assert.Equal(t, 6, c.Get("logs_config.compression_level")) assert.Equal(t, "https://trace.agent.datadoghq.com", c.Get("apm_config.apm_dd_url")) assert.Equal(t, false, c.Get("apm_config.receiver_enabled")) - assert.Equal(t, true, c.Get("otlp_config.traces.span_name_as_resource_name")) + assert.Equal(t, false, c.Get("otlp_config.traces.span_name_as_resource_name")) assert.Equal(t, []string{"enable_receive_resource_spans_v2", "enable_operation_and_resource_name_logic_v2", "enable_otlp_compute_top_level_by_span_kind"}, c.Get("apm_config.features")) // log_level from datadog.yaml takes precedence -> more verbose diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index a414674620e44..d7958c6dabc59 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/core/hostname/remotehostnameimpl" log "github.com/DataDog/datadog-agent/comp/core/log/def" + logfx "github.com/DataDog/datadog-agent/comp/core/log/fx" logtracefx "github.com/DataDog/datadog-agent/comp/core/log/fx-trace" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" @@ -106,8 +107,20 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Provide(func() coreconfig.Component { return acfg }), + fx.Provide(func(_ coreconfig.Component) log.Params { + return log.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile) + }), + logfx.Module(), + fetchonlyimpl.Module(), + // TODO: don't rely on this pattern; remove this `ModuleWithParams` thing + // and instead adapt OptionalModule to allow parameter passing naturally. + // See: https://github.com/DataDog/datadog-agent/pull/28386 + configsyncimpl.ModuleWithParams(), + fx.Provide(func() configsyncimpl.Params { + return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) + }), converterfx.Module(), - fx.Provide(func(cp converter.Component) confmap.Converter { + fx.Provide(func(cp converter.Component, _ configsync.Component) confmap.Converter { return cp }), collectorcontribFx.Module(), @@ -178,10 +191,10 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Invoke(func(_ collectordef.Component, _ defaultforwarder.Forwarder, _ optional.Option[logsagentpipeline.Component]) { }), - // TODO: don't rely on this pattern; remove this `OptionalModuleWithParams` thing + // TODO: don't rely on this pattern; remove this `ModuleWithParams` thing // and instead adapt OptionalModule to allow parameter passing naturally. // See: https://github.com/DataDog/datadog-agent/pull/28386 - configsyncimpl.OptionalModuleWithParams(), + configsyncimpl.ModuleWithParams(), fx.Provide(func() configsyncimpl.Params { return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) }), @@ -205,7 +218,7 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, // TODO: consider adding configsync.Component as an explicit dependency for traceconfig // to avoid this sort of dependency tree hack. - fx.Provide(func(deps traceconfig.Dependencies, _ optional.Option[configsync.Component]) (traceconfig.Component, error) { + fx.Provide(func(deps traceconfig.Dependencies, _ configsync.Component) (traceconfig.Component, error) { // TODO: this would be much better if we could leverage traceconfig.Module // Must add a new parameter to traconfig.Module to handle this. return traceconfig.NewConfig(deps) @@ -223,13 +236,13 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, // ForwarderBundle returns the fx.Option for the forwarder bundle. // TODO: cleanup the forwarder instantiation with fx. -// This is a bit of a hack because we need to enforce optional.Option[configsync.Component] +// This is a bit of a hack because we need to enforce configsync.Component // is passed to newForwarder to enforce the correct instantiation order. Currently, the // new forwarder.BundleWithProvider makes a few assumptions in its generic prototype, and // this is the current workaround to leverage it. func ForwarderBundle() fx.Option { return defaultforwarder.ModulWithOptionTMP( - fx.Provide(func(_ optional.Option[configsync.Component]) defaultforwarder.Params { + fx.Provide(func(_ configsync.Component) defaultforwarder.Params { return defaultforwarder.NewParams() })) } diff --git a/cmd/process-agent/command/main_common.go b/cmd/process-agent/command/main_common.go index dcb9287e74f04..4d6b33d5ad831 100644 --- a/cmd/process-agent/command/main_common.go +++ b/cmd/process-agent/command/main_common.go @@ -65,7 +65,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/fxutil/logging" "github.com/DataDog/datadog-agent/pkg/util/log" - "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/version" ) @@ -156,7 +155,7 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error { fetchonlyimpl.Module(), // Provide configsync module - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Provide autoexit module autoexitimpl.Module(), @@ -215,7 +214,7 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error { _ expvars.Component, _ apiserver.Component, cfg config.Component, - _ optional.Option[configsync.Component], + _ configsync.Component, // TODO: This is needed by the container-provider which is not currently a component. // We should ensure the tagger is a dependency when converting to a component. _ tagger.Component, diff --git a/cmd/process-agent/subcommands/config/config.go b/cmd/process-agent/subcommands/config/config.go index 1acd0ec15e5e2..f7a9996ac10ab 100644 --- a/cmd/process-agent/subcommands/config/config.go +++ b/cmd/process-agent/subcommands/config/config.go @@ -181,6 +181,11 @@ func getConfigValue(deps dependencies, args []string) error { } func getClient(cfg model.Reader) (settings.Client, error) { + err := util.SetAuthToken(cfg) + if err != nil { + return nil, err + } + httpClient := apiutil.GetClient(false) ipcAddress, err := pkgconfigsetup.GetIPCAddress(pkgconfigsetup.Datadog()) diff --git a/cmd/process-agent/subcommands/status/status.go b/cmd/process-agent/subcommands/status/status.go index 2660ee8fb71b4..f98877e1979a2 100644 --- a/cmd/process-agent/subcommands/status/status.go +++ b/cmd/process-agent/subcommands/status/status.go @@ -21,6 +21,7 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" compStatus "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/process" + "github.com/DataDog/datadog-agent/pkg/api/util" apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/collector/python" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -28,8 +29,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) -var httpClient = apiutil.GetClient(false) - const ( notRunning = ` ============= @@ -111,6 +110,7 @@ func writeError(log log.Component, w io.Writer, e error) { } func fetchStatus(statusURL string) ([]byte, error) { + httpClient := apiutil.GetClient(false) body, err := apiutil.DoGet(httpClient, statusURL, apiutil.LeaveConnectionOpen) if err != nil { return nil, status.NewConnectionError(err) @@ -148,6 +148,11 @@ func runStatus(deps dependencies) error { return err } + err = util.SetAuthToken(deps.Config) + if err != nil { + return err + } + getAndWriteStatus(deps.Log, statusURL, os.Stdout) return nil } diff --git a/cmd/process-agent/subcommands/taggerlist/tagger_list.go b/cmd/process-agent/subcommands/taggerlist/tagger_list.go index e7ec238efdeea..235a5a2b958df 100644 --- a/cmd/process-agent/subcommands/taggerlist/tagger_list.go +++ b/cmd/process-agent/subcommands/taggerlist/tagger_list.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/tagger/api" + "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -58,6 +59,10 @@ func taggerList(deps dependencies) error { return err } + err = util.SetAuthToken(deps.Config) + if err != nil { + return err + } return api.GetTaggerList(color.Output, taggerURL) } diff --git a/cmd/security-agent/api/server.go b/cmd/security-agent/api/server.go index e776628e5ccd4..307a4e4c6cd43 100644 --- a/cmd/security-agent/api/server.go +++ b/cmd/security-agent/api/server.go @@ -12,9 +12,6 @@ package api import ( "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" stdLog "log" "net" "net/http" @@ -23,10 +20,10 @@ import ( "github.com/gorilla/mux" "github.com/DataDog/datadog-agent/cmd/security-agent/api/agent" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/core/settings" "github.com/DataDog/datadog-agent/comp/core/status" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" - "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -35,19 +32,21 @@ import ( // Server implements security agent API server type Server struct { - listener net.Listener - agent *agent.Agent + listener net.Listener + agent *agent.Agent + tlsConfig *tls.Config } // NewServer creates a new Server instance -func NewServer(statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component) (*Server, error) { +func NewServer(statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) (*Server, error) { listener, err := newListener() if err != nil { return nil, err } return &Server{ - listener: listener, - agent: agent.NewAgent(statusComponent, settings, wmeta), + listener: listener, + agent: agent.NewAgent(statusComponent, settings, wmeta), + tlsConfig: at.GetTLSServerConfig(), }, nil } @@ -62,43 +61,16 @@ func (s *Server) Start() error { // Validate token for every request r.Use(validateToken) - err := util.CreateAndSetAuthToken(pkgconfigsetup.Datadog()) - if err != nil { - return err - } - - hosts := []string{"127.0.0.1", "localhost"} - _, rootCertPEM, rootKey, err := security.GenerateRootCert(hosts, 2048) - if err != nil { - return fmt.Errorf("unable to start TLS server") - } - - // PEM encode the private key - rootKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), - }) - - // Create a TLS cert using the private key and certificate - rootTLSCert, err := tls.X509KeyPair(rootCertPEM, rootKeyPEM) - if err != nil { - return fmt.Errorf("invalid key pair: %v", err) - } - - tlsConfig := tls.Config{ - Certificates: []tls.Certificate{rootTLSCert}, - MinVersion: tls.VersionTLS13, - } - // Use a stack depth of 4 on top of the default one to get a relevant filename in the stdlib logWriter, _ := pkglogsetup.NewLogWriter(4, log.ErrorLvl) srv := &http.Server{ Handler: r, ErrorLog: stdLog.New(logWriter, "Error from the agent http API server: ", 0), // log errors to seelog, - TLSConfig: &tlsConfig, + TLSConfig: s.tlsConfig, WriteTimeout: pkgconfigsetup.Datadog().GetDuration("server_timeout") * time.Second, } - tlsListener := tls.NewListener(s.listener, &tlsConfig) + tlsListener := tls.NewListener(s.listener, s.tlsConfig) go srv.Serve(tlsListener) //nolint:errcheck return nil diff --git a/cmd/security-agent/main_windows.go b/cmd/security-agent/main_windows.go index b81223e6aebc1..f6a2e3aedf28f 100644 --- a/cmd/security-agent/main_windows.go +++ b/cmd/security-agent/main_windows.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/start" "github.com/DataDog/datadog-agent/comp/agent/autoexit" "github.com/DataDog/datadog-agent/comp/agent/autoexit/autoexitimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" @@ -54,7 +55,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/util/startstop" "github.com/DataDog/datadog-agent/pkg/util/winutil/servicemain" ) @@ -91,10 +91,11 @@ func (s *service) Run(svcctx context.Context) error { params := &cliParams{} err := fxutil.OneShot( func(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, - telemetry telemetry.Component, _ workloadmeta.Component, _ *cliParams, statusComponent status.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component) error { + telemetry telemetry.Component, _ workloadmeta.Component, _ *cliParams, statusComponent status.Component, _ autoexit.Component, + settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) error { defer start.StopAgent(log) - err := start.RunAgent(log, config, telemetry, statusComponent, settings, wmeta) + err := start.RunAgent(log, config, telemetry, statusComponent, settings, wmeta, at) if err != nil { if errors.Is(err, start.ErrAllComponentsDisabled) { // If all components are disabled, we should exit cleanly @@ -164,9 +165,9 @@ func (s *service) Run(svcctx context.Context) error { statusimpl.Module(), fetchonlyimpl.Module(), - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Force the instantiation of the component - fx.Invoke(func(_ optional.Option[configsync.Component]) {}), + fx.Invoke(func(_ configsync.Component) {}), autoexitimpl.Module(), fx.Provide(func(c config.Component) settings.Params { return settings.Params{ diff --git a/cmd/security-agent/subcommands/runtime/activity_dump.go b/cmd/security-agent/subcommands/runtime/activity_dump.go index 35907553045fb..0f15351d72e02 100644 --- a/cmd/security-agent/subcommands/runtime/activity_dump.go +++ b/cmd/security-agent/subcommands/runtime/activity_dump.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" "os" + "time" "github.com/spf13/cobra" "go.uber.org/fx" @@ -173,7 +174,7 @@ func generateDumpCommands(globalParams *command.GlobalParams) []*cobra.Command { activityDumpGenerateDumpCmd.Flags().StringVar( &cliParams.timeout, "timeout", - "1m", + "", "timeout for the activity dump", ) activityDumpGenerateDumpCmd.Flags().BoolVar( @@ -185,7 +186,7 @@ func generateDumpCommands(globalParams *command.GlobalParams) []*cobra.Command { activityDumpGenerateDumpCmd.Flags().StringVar( &cliParams.localStorageDirectory, "output", - "/tmp/activity_dumps/", + "", "local storage output directory", ) activityDumpGenerateDumpCmd.Flags().BoolVar( @@ -472,6 +473,12 @@ func generateActivityDump(_ log.Component, _ config.Component, _ secrets.Compone return err } + if activityDumpArgs.timeout != "" { + if _, err = time.ParseDuration(activityDumpArgs.timeout); err != nil { + return err + } + } + output, err := client.GenerateActivityDump(&api.ActivityDumpParams{ ContainerID: activityDumpArgs.containerID, CGroupID: activityDumpArgs.cgroupID, diff --git a/cmd/security-agent/subcommands/start/command.go b/cmd/security-agent/subcommands/start/command.go index 12a93fc4560ff..933a44f4a0135 100644 --- a/cmd/security-agent/subcommands/start/command.go +++ b/cmd/security-agent/subcommands/start/command.go @@ -29,6 +29,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/runtime" "github.com/DataDog/datadog-agent/comp/agent/autoexit" "github.com/DataDog/datadog-agent/comp/agent/autoexit/autoexitimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" @@ -65,7 +66,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util/coredump" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/util/profiling" "github.com/DataDog/datadog-agent/pkg/util/startstop" "github.com/DataDog/datadog-agent/pkg/version" @@ -173,9 +173,9 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { }), statusimpl.Module(), fetchonlyimpl.Module(), - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Force the instantiation of the component - fx.Invoke(func(_ optional.Option[configsync.Component]) {}), + fx.Invoke(func(_ configsync.Component) {}), autoexitimpl.Module(), fx.Supply(pidimpl.NewParams(params.pidfilePath)), fx.Provide(func(c config.Component) settings.Params { @@ -201,10 +201,10 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { // TODO(components): note how workloadmeta is passed anonymously, it is still required as it is used // as a global. This should eventually be fixed and all workloadmeta interactions should be via the // injected instance. -func start(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, telemetry telemetry.Component, statusComponent status.Component, _ pid.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component) error { +func start(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, telemetry telemetry.Component, statusComponent status.Component, _ pid.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) error { defer StopAgent(log) - err := RunAgent(log, config, telemetry, statusComponent, settings, wmeta) + err := RunAgent(log, config, telemetry, statusComponent, settings, wmeta, at) if errors.Is(err, ErrAllComponentsDisabled) || errors.Is(err, errNoAPIKeyConfigured) { return nil } @@ -256,7 +256,7 @@ var ErrAllComponentsDisabled = errors.New("all security-agent component are disa var errNoAPIKeyConfigured = errors.New("no API key configured") // RunAgent initialized resources and starts API server -func RunAgent(log log.Component, config config.Component, telemetry telemetry.Component, statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component) (err error) { +func RunAgent(log log.Component, config config.Component, telemetry telemetry.Component, statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) (err error) { if err := coredump.Setup(config); err != nil { log.Warnf("Can't setup core dumps: %v, core dumps might not be available after a crash", err) } @@ -299,7 +299,7 @@ func RunAgent(log log.Component, config config.Component, telemetry telemetry.Co } }() - srv, err = api.NewServer(statusComponent, settings, wmeta) + srv, err = api.NewServer(statusComponent, settings, wmeta, at) if err != nil { return log.Errorf("Error while creating api server, exiting: %v", err) } diff --git a/cmd/system-probe/api/debug/handlers_linux.go b/cmd/system-probe/api/debug/handlers_linux.go index d2bd7dfbd5f48..07ba06c49354f 100644 --- a/cmd/system-probe/api/debug/handlers_linux.go +++ b/cmd/system-probe/api/debug/handlers_linux.go @@ -17,19 +17,18 @@ import ( "time" ) -// HandleSelinuxSestatus reports the output of sestatus as an http result -func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - cmd := exec.CommandContext(ctx, "sestatus") +// handleCommand runs commandName with the provided arguments and writes it to the HTTP response. +// If the command exits with a failure or doesn't exist in the PATH, it will still 200 but report the failure. +// Any other kind of error will 500. +func handleCommand(ctx context.Context, w http.ResponseWriter, commandName string, args ...string) { + cmd := exec.CommandContext(ctx, commandName, args...) output, err := cmd.CombinedOutput() var execError *exec.Error var exitErr *exec.ExitError if err != nil { - // don't 500 for ExitErrors etc, to report "normal" failures to the selinux_sestatus.log file + // don't 500 for ExitErrors etc, to report "normal" failures to the flare log file if !errors.As(err, &execError) && !errors.As(err, &exitErr) { w.WriteHeader(500) } @@ -39,3 +38,19 @@ func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { w.Write(output) } + +// HandleSelinuxSestatus reports the output of sestatus as an http result +func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "sestatus") +} + +// HandleSelinuxSemoduleList reports the output of semodule -l as an http result +func HandleSelinuxSemoduleList(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "semodule", "-l") +} diff --git a/cmd/system-probe/api/debug/handlers_nolinux.go b/cmd/system-probe/api/debug/handlers_nolinux.go index 1475d821c1e6e..246f4a3a7c78a 100644 --- a/cmd/system-probe/api/debug/handlers_nolinux.go +++ b/cmd/system-probe/api/debug/handlers_nolinux.go @@ -18,3 +18,9 @@ func HandleSelinuxSestatus(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) io.WriteString(w, "HandleSelinuxSestatus is not supported on this platform") } + +// HandleSelinuxSemoduleList is not supported +func HandleSelinuxSemoduleList(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(500) + io.WriteString(w, "HandleSelinuxSemoduleList is not supported on this platform") +} diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index d81007a0c8f0d..f4d9e85522d91 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -60,6 +60,7 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet if runtime.GOOS == "linux" { mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) mux.HandleFunc("/debug/selinux_sestatus", debug.HandleSelinuxSestatus) + mux.HandleFunc("/debug/selinux_semodule_list", debug.HandleSelinuxSemoduleList) } go func() { diff --git a/cmd/system-probe/modules/gpu.go b/cmd/system-probe/modules/gpu.go index 7ec273e7c2869..c7dbac874dccd 100644 --- a/cmd/system-probe/modules/gpu.go +++ b/cmd/system-probe/modules/gpu.go @@ -55,6 +55,7 @@ var GPUMonitoring = module.Factory{ //(https://github.com/NVIDIA/go-nvml/blob/main/pkg/nvml/lib.go#L30) NvmlLib: nvml.New(nvml.WithLibraryPath(c.NVMLLibraryPath)), ProcessMonitor: processEventConsumer, + WorkloadMeta: deps.WMeta, } ret := probeDeps.NvmlLib.Init() diff --git a/cmd/trace-agent/config/remote/config.go b/cmd/trace-agent/config/remote/config.go index d2c511a4586e7..6b650364743a7 100644 --- a/cmd/trace-agent/config/remote/config.go +++ b/cmd/trace-agent/config/remote/config.go @@ -43,7 +43,7 @@ func putBuffer(buffer *bytes.Buffer) { // ConfigHandler is the HTTP handler for configs func ConfigHandler(r *api.HTTPReceiver, cf rcclient.ConfigFetcher, cfg *config.AgentConfig, statsd statsd.ClientInterface, timing timing.Reporter) http.Handler { - cidProvider := api.NewIDProvider(cfg.ContainerProcRoot) + cidProvider := api.NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo) return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer timing.Since("datadog.trace_agent.receiver.config_process_ms", time.Now()) tags := r.TagStats(api.V07, req.Header, "").AsTags() diff --git a/cmd/trace-agent/subcommands/run/command.go b/cmd/trace-agent/subcommands/run/command.go index 1d0788501ffae..4526101362de1 100644 --- a/cmd/trace-agent/subcommands/run/command.go +++ b/cmd/trace-agent/subcommands/run/command.go @@ -113,9 +113,9 @@ func runTraceAgentProcess(ctx context.Context, cliParams *Params, defaultConfPat zstdfx.Module(), trace.Bundle(), fetchonlyimpl.Module(), - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Force the instantiation of the components - fx.Invoke(func(_ traceagent.Component, _ optional.Option[configsync.Component], _ autoexit.Component) {}), + fx.Invoke(func(_ traceagent.Component, _ configsync.Component, _ autoexit.Component) {}), ) if err != nil && errors.Is(err, traceagentimpl.ErrAgentDisabled) { return nil diff --git a/comp/api/api/apiimpl/api_test.go b/comp/api/api/apiimpl/api_test.go index d92e8f1e4fa48..b248dc9531330 100644 --- a/comp/api/api/apiimpl/api_test.go +++ b/comp/api/api/apiimpl/api_test.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl/observability" api "github.com/DataDog/datadog-agent/comp/api/api/def" - "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" "github.com/DataDog/datadog-agent/comp/collector/collector" "github.com/DataDog/datadog-agent/comp/core/autodiscovery" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/autodiscoveryimpl" @@ -76,7 +76,7 @@ func getTestAPIServer(t *testing.T, params config.MockParams) testdeps { demultiplexerimpl.MockModule(), fx.Supply(optional.NewNoneOption[rcservice.Component]()), fx.Supply(optional.NewNoneOption[rcservicemrf.Component]()), - fetchonlyimpl.MockModule(), + createandfetchimpl.Module(), fx.Supply(context.Background()), taggermock.Module(), fx.Provide(func(mock taggermock.Mock) tagger.Component { @@ -166,7 +166,6 @@ func TestStartBothServersWithObservability(t *testing.T) { req, err := http.NewRequest(http.MethodGet, url, nil) require.NoError(t, err) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authTokenValue)) resp, err := util.GetClient(false).Do(req) require.NoError(t, err) defer resp.Body.Close() diff --git a/comp/api/api/apiimpl/grpc.go b/comp/api/api/apiimpl/grpc.go index 5f88cdb722163..6272a7f00fe05 100644 --- a/comp/api/api/apiimpl/grpc.go +++ b/comp/api/api/apiimpl/grpc.go @@ -74,6 +74,12 @@ func (s *serverSecure) TaggerStreamEntities(req *pb.StreamTagsRequest, srv pb.Ag return s.taggerServer.TaggerStreamEntities(req, srv) } +// TaggerGenerateContainerIDFromOriginInfo generates a container ID from the Origin Info. +// This function takes an Origin Info but only uses the ExternalData part of it, this is done for backward compatibility. +func (s *serverSecure) TaggerGenerateContainerIDFromOriginInfo(ctx context.Context, req *pb.GenerateContainerIDFromOriginInfoRequest) (*pb.GenerateContainerIDFromOriginInfoResponse, error) { + return s.taggerServer.TaggerGenerateContainerIDFromOriginInfo(ctx, req) +} + func (s *serverSecure) TaggerFetchEntity(ctx context.Context, req *pb.FetchEntityRequest) (*pb.FetchEntityResponse, error) { return s.taggerServer.TaggerFetchEntity(ctx, req) } diff --git a/comp/api/api/apiimpl/security.go b/comp/api/api/apiimpl/security.go index 4587871aecec3..4b8b17d2ba33d 100644 --- a/comp/api/api/apiimpl/security.go +++ b/comp/api/api/apiimpl/security.go @@ -6,16 +6,10 @@ package apiimpl import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" + "crypto/subtle" "errors" - "fmt" "net/http" - "runtime" - "strings" - "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -34,7 +28,7 @@ func validateToken(next http.Handler) http.Handler { // parseToken parses the token and validate it for our gRPC API, it returns an empty // struct and an error or nil func parseToken(token string) (interface{}, error) { - if token != util.GetAuthToken() { + if subtle.ConstantTimeCompare([]byte(token), []byte(util.GetAuthToken())) == 0 { return struct{}{}, errors.New("Invalid session token") } @@ -43,44 +37,3 @@ func parseToken(token string) (interface{}, error) { // type. return struct{}{}, nil } - -func buildSelfSignedKeyPair(additionalHostIdentities ...string) ([]byte, []byte) { - hosts := []string{"127.0.0.1", "localhost", "::1"} - hosts = append(hosts, additionalHostIdentities...) - _, rootCertPEM, rootKey, err := security.GenerateRootCert(hosts, 2048) - if err != nil { - return nil, nil - } - - // PEM encode the private key - rootKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), - }) - - // Create and return TLS private cert and key - return rootCertPEM, rootKeyPEM -} - -func initializeTLS(additionalHostIdentities ...string) (*tls.Certificate, *x509.CertPool, error) { - // print the caller to identify what is calling this function - if _, file, line, ok := runtime.Caller(1); ok { - log.Infof("[%s:%d] Initializing TLS certificates for hosts %v", file, line, strings.Join(additionalHostIdentities, ", ")) - } - - cert, key := buildSelfSignedKeyPair(additionalHostIdentities...) - if cert == nil { - return nil, nil, errors.New("unable to generate certificate") - } - pair, err := tls.X509KeyPair(cert, key) - if err != nil { - return nil, nil, fmt.Errorf("unable to generate TLS key pair: %v", err) - } - - tlsCertPool := x509.NewCertPool() - ok := tlsCertPool.AppendCertsFromPEM(cert) - if !ok { - return nil, nil, fmt.Errorf("unable to add new certificate to pool") - } - - return &pair, tlsCertPool, nil -} diff --git a/comp/api/api/apiimpl/server.go b/comp/api/api/apiimpl/server.go index 762a67b8a4a63..6def35781eb1a 100644 --- a/comp/api/api/apiimpl/server.go +++ b/comp/api/api/apiimpl/server.go @@ -11,6 +11,7 @@ import ( stdLog "log" "net" "net/http" + "strconv" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl/observability" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -47,34 +48,11 @@ func (server *apiServer) startServers() error { return fmt.Errorf("unable to get IPC address and port: %v", err) } - additionalHostIdentities := []string{apiAddr} - - ipcServerHost, ipcServerHostPort, ipcServerEnabled := getIPCServerAddressPort() - if ipcServerEnabled { - additionalHostIdentities = append(additionalHostIdentities, ipcServerHost) - } - - tlsKeyPair, tlsCertPool, err := initializeTLS(additionalHostIdentities...) - if err != nil { - return fmt.Errorf("unable to initialize TLS: %v", err) - } - - // tls.Config is written to when serving, so it has to be cloned for each server - tlsConfig := func() *tls.Config { - return &tls.Config{ - Certificates: []tls.Certificate{*tlsKeyPair}, - NextProtos: []string{"h2"}, - MinVersion: tls.VersionTLS12, - } - } - tmf := observability.NewTelemetryMiddlewareFactory(server.telemetry) // start the CMD server if err := server.startCMDServer( apiAddr, - tlsConfig(), - tlsCertPool, tmf, server.cfg, ); err != nil { @@ -82,8 +60,11 @@ func (server *apiServer) startServers() error { } // start the IPC server - if ipcServerEnabled { - if err := server.startIPCServer(ipcServerHostPort, tlsConfig(), tmf); err != nil { + if ipcServerPort := server.cfg.GetInt("agent_ipc.port"); ipcServerPort > 0 { + ipcServerHost := server.cfg.GetString("agent_ipc.host") + ipcServerHostPort := net.JoinHostPort(ipcServerHost, strconv.Itoa(ipcServerPort)) + + if err := server.startIPCServer(ipcServerHostPort, tmf); err != nil { // if we fail to start the IPC server, we should stop the CMD server server.stopServers() return fmt.Errorf("unable to start IPC API server: %v", err) diff --git a/comp/api/api/apiimpl/server_cmd.go b/comp/api/api/apiimpl/server_cmd.go index 2b806aa8726e3..2eca0fe7abd09 100644 --- a/comp/api/api/apiimpl/server_cmd.go +++ b/comp/api/api/apiimpl/server_cmd.go @@ -7,8 +7,6 @@ package apiimpl import ( "context" - "crypto/tls" - "crypto/x509" "fmt" "net/http" "time" @@ -35,8 +33,6 @@ const cmdServerShortName string = "CMD" func (server *apiServer) startCMDServer( cmdAddr string, - tlsConfig *tls.Config, - tlsCertPool *x509.CertPool, tmf observability.TelemetryMiddlewareFactory, cfg config.Component, ) (err error) { @@ -54,7 +50,7 @@ func (server *apiServer) startCMDServer( maxMessageSize := cfg.GetInt("cluster_agent.cluster_tagger.grpc_max_message_size") opts := []grpc.ServerOption{ - grpc.Creds(credentials.NewClientTLSFromCert(tlsCertPool, cmdAddr)), + grpc.Creds(credentials.NewTLS(server.authToken.GetTLSServerConfig())), grpc.StreamInterceptor(grpc_auth.StreamServerInterceptor(authInterceptor)), grpc.UnaryInterceptor(grpc_auth.UnaryServerInterceptor(authInterceptor)), grpc.MaxRecvMsgSize(maxMessageSize), @@ -79,11 +75,7 @@ func (server *apiServer) startCMDServer( autodiscovery: server.autoConfig, }) - dcreds := credentials.NewTLS(&tls.Config{ - ServerName: cmdAddr, - RootCAs: tlsCertPool, - }) - dopts := []grpc.DialOption{grpc.WithTransportCredentials(dcreds)} + dopts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(server.authToken.GetTLSClientConfig()))} // starting grpc gateway ctx := context.Background() @@ -133,7 +125,7 @@ func (server *apiServer) startCMDServer( srv := grpcutil.NewMuxedGRPCServer( cmdAddr, - tlsConfig, + server.authToken.GetTLSServerConfig(), s, grpcutil.TimeoutHandlerFunc(cmdMuxHandler, time.Duration(pkgconfigsetup.Datadog().GetInt64("server_timeout"))*time.Second), ) diff --git a/comp/api/api/apiimpl/server_ipc.go b/comp/api/api/apiimpl/server_ipc.go index 10e17509f75e3..634ec1810f35a 100644 --- a/comp/api/api/apiimpl/server_ipc.go +++ b/comp/api/api/apiimpl/server_ipc.go @@ -6,7 +6,6 @@ package apiimpl import ( - "crypto/tls" "net/http" "time" @@ -18,7 +17,7 @@ import ( const ipcServerName string = "IPC API Server" const ipcServerShortName string = "IPC" -func (server *apiServer) startIPCServer(ipcServerAddr string, tlsConfig *tls.Config, tmf observability.TelemetryMiddlewareFactory) (err error) { +func (server *apiServer) startIPCServer(ipcServerAddr string, tmf observability.TelemetryMiddlewareFactory) (err error) { server.ipcListener, err = getListener(ipcServerAddr) if err != nil { return err @@ -39,7 +38,7 @@ func (server *apiServer) startIPCServer(ipcServerAddr string, tlsConfig *tls.Con ipcServer := &http.Server{ Addr: ipcServerAddr, Handler: http.TimeoutHandler(ipcMuxHandler, time.Duration(pkgconfigsetup.Datadog().GetInt64("server_timeout"))*time.Second, "timeout"), - TLSConfig: tlsConfig, + TLSConfig: server.authToken.GetTLSServerConfig(), } startServer(server.ipcListener, ipcServer, ipcServerName) diff --git a/comp/api/authtoken/go.mod b/comp/api/authtoken/go.mod index ca4e617286f9e..24081f4824780 100644 --- a/comp/api/authtoken/go.mod +++ b/comp/api/authtoken/go.mod @@ -84,7 +84,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/comp/api/authtoken/go.sum b/comp/api/authtoken/go.sum index 9694f5d5e2355..dd16364891695 100644 --- a/comp/api/authtoken/go.sum +++ b/comp/api/authtoken/go.sum @@ -87,8 +87,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go index d8631536c3a44..6098dcf98f629 100644 --- a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go +++ b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go @@ -168,11 +168,22 @@ func (wcd *AgentCrashDetect) Run() error { } log.Infof("Sending crash: %v", formatText(crash)) - lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), wcd.tconfig.TelemetryConfig.Endpoints, "ddnpm", true) + lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), toTelemEndpoints(wcd.tconfig.TelemetryConfig.Endpoints), "ddnpm", true) lts.SendLog("WARN", formatText(crash)) return nil } +func toTelemEndpoints(endpoints []*traceconfig.Endpoint) []*internaltelemetry.Endpoint { + telemEndpoints := make([]*internaltelemetry.Endpoint, 0, len(endpoints)) + for _, e := range endpoints { + telemEndpoints = append(telemEndpoints, &internaltelemetry.Endpoint{ + Host: e.Host, + APIKey: e.APIKey, + }) + } + return telemEndpoints +} + func newAgentCrashComponent(deps dependencies) agentcrashdetect.Component { instance := &agentCrashComponent{} instance.tconfig = deps.TConfig.Object() diff --git a/comp/core/config/go.mod b/comp/core/config/go.mod index a8fc87fffecf0..00be19b3b9ba7 100644 --- a/comp/core/config/go.mod +++ b/comp/core/config/go.mod @@ -75,7 +75,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/comp/core/config/go.sum b/comp/core/config/go.sum index 49f42d5d3e0a6..323c4fa804e83 100644 --- a/comp/core/config/go.sum +++ b/comp/core/config/go.sum @@ -88,8 +88,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/core/configsync/configsyncimpl/module.go b/comp/core/configsync/configsyncimpl/module.go index 0b77c1370de20..a89bbacc91197 100644 --- a/comp/core/configsync/configsyncimpl/module.go +++ b/comp/core/configsync/configsyncimpl/module.go @@ -22,7 +22,6 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" ) type dependencies struct { @@ -35,20 +34,20 @@ type dependencies struct { SyncParams Params } -// OptionalModule defines the fx options for this component. -func OptionalModule() fxutil.Module { +// Module defines the fx options for this component. +func Module() fxutil.Module { return fxutil.Component( - fx.Provide(newOptionalConfigSync), + fx.Provide(newComponent), fx.Supply(Params{}), ) } -// OptionalModuleWithParams defines the fx options for this component, but +// ModuleWithParams defines the fx options for this component, but // requires additionally specifying custom Params from the fx App, to be // passed to the constructor. -func OptionalModuleWithParams() fxutil.Module { +func ModuleWithParams() fxutil.Module { return fxutil.Component( - fx.Provide(newOptionalConfigSync), + fx.Provide(newComponent), ) } @@ -61,19 +60,21 @@ type configSync struct { client *http.Client connected bool ctx context.Context + enabled bool } -// newOptionalConfigSync checks if the component was enabled as per the config, and returns an optional.Option -func newOptionalConfigSync(deps dependencies) optional.Option[configsync.Component] { +// newComponent checks if the component was enabled as per the config and return a enable/disabled configsync +func newComponent(deps dependencies) configsync.Component { agentIPCPort := deps.Config.GetInt("agent_ipc.port") configRefreshIntervalSec := deps.Config.GetInt("agent_ipc.config_refresh_interval") if agentIPCPort <= 0 || configRefreshIntervalSec <= 0 { - return optional.NewNoneOption[configsync.Component]() + deps.Log.Infof("configsync disabled (agent_ipc.port: %d | agent_ipc.config_refresh_interval: %d)", agentIPCPort, configRefreshIntervalSec) + return configSync{} } - configSync := newConfigSync(deps, agentIPCPort, configRefreshIntervalSec) - return optional.NewOption(configSync) + deps.Log.Infof("configsync enabled (agent_ipc '%s:%d' | agent_ipc.config_refresh_interval: %d)", deps.Config.GetString("agent_ipc.host"), agentIPCPort, configRefreshIntervalSec) + return newConfigSync(deps, agentIPCPort, configRefreshIntervalSec) } // newConfigSync creates a new configSync component. @@ -98,6 +99,7 @@ func newConfigSync(deps dependencies, agentIPCPort int, configRefreshIntervalSec url: url, client: client, ctx: ctx, + enabled: true, } if deps.SyncParams.OnInit { diff --git a/comp/core/configsync/configsyncimpl/module_integration_test.go b/comp/core/configsync/configsyncimpl/module_integration_test.go index e4cfa8ec1ba63..5d0fde62a449a 100644 --- a/comp/core/configsync/configsyncimpl/module_integration_test.go +++ b/comp/core/configsync/configsyncimpl/module_integration_test.go @@ -22,7 +22,6 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/configsync" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" ) func TestOptionalModule(t *testing.T) { @@ -44,16 +43,14 @@ func TestOptionalModule(t *testing.T) { "agent_ipc.port": port, "agent_ipc.config_refresh_interval": 1, } - csopt := fxutil.Test[optional.Option[configsync.Component]](t, fx.Options( + comp := fxutil.Test[configsync.Component](t, fx.Options( core.MockBundle(), fetchonlyimpl.Module(), - OptionalModule(), + Module(), fx.Populate(&cfg), fx.Replace(config.MockParams{Overrides: overrides}), )) - - _, ok := csopt.Get() - require.True(t, ok) + require.True(t, comp.(configSync).enabled) require.EventuallyWithT(t, func(t *assert.CollectT) { assert.Equal(t, "value1", cfg.Get("key1")) diff --git a/comp/core/configsync/configsyncimpl/module_test.go b/comp/core/configsync/configsyncimpl/module_test.go index 076288737a65e..8051e4cce793c 100644 --- a/comp/core/configsync/configsyncimpl/module_test.go +++ b/comp/core/configsync/configsyncimpl/module_test.go @@ -8,34 +8,30 @@ package configsyncimpl import ( "testing" - "github.com/stretchr/testify/require" - pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" + "github.com/stretchr/testify/assert" ) -func TestNewOptionalConfigSync(t *testing.T) { +func TestNewConfigSync(t *testing.T) { t.Run("enabled", func(t *testing.T) { deps := makeDeps(t) deps.Config.Set("agent_ipc.port", 1234, pkgconfigmodel.SourceFile) deps.Config.Set("agent_ipc.config_refresh_interval", 30, pkgconfigmodel.SourceFile) - optConfigSync := newOptionalConfigSync(deps) - _, ok := optConfigSync.Get() - require.True(t, ok) + comp := newComponent(deps) + assert.True(t, comp.(configSync).enabled) }) t.Run("disabled ipc port zero", func(t *testing.T) { deps := makeDeps(t) deps.Config.Set("agent_ipc.port", 0, pkgconfigmodel.SourceFile) - optConfigSync := newOptionalConfigSync(deps) - _, ok := optConfigSync.Get() - require.False(t, ok) + comp := newComponent(deps) + assert.False(t, comp.(configSync).enabled) }) t.Run("disabled config refresh interval zero", func(t *testing.T) { deps := makeDeps(t) deps.Config.Set("agent_ipc.config_refresh_interval", 0, pkgconfigmodel.SourceFile) - optConfigSync := newOptionalConfigSync(deps) - _, ok := optConfigSync.Get() - require.False(t, ok) + comp := newComponent(deps) + assert.False(t, comp.(configSync).enabled) }) } diff --git a/comp/core/flare/helpers/builder_test.go b/comp/core/flare/helpers/builder_test.go index 26508575eef92..ccc544eb6ecb9 100644 --- a/comp/core/flare/helpers/builder_test.go +++ b/comp/core/flare/helpers/builder_test.go @@ -213,8 +213,8 @@ func TestAddFileYamlDetection(t *testing.T) { - abcdef - abcdef`) redacted := `instances: -- host: 127.0.0.1 - token: "********"` + - host: 127.0.0.1 + token: "********"` fb.AddFile("test.yaml", clear) assertFileContent(t, fb, redacted, "test.yaml") @@ -297,8 +297,8 @@ func TestCopyFileYamlDetection(t *testing.T) { - abcdef - abcdef`) redacted := `instances: -- host: 127.0.0.1 - token: "********"` + - host: 127.0.0.1 + token: "********"` path1 := filepath.Join(t.TempDir(), "test.yaml") os.WriteFile(path1, []byte(input), os.ModePerm) diff --git a/comp/core/flare/providers.go b/comp/core/flare/providers.go index edb2c7ac86520..69ceae5ccf76f 100644 --- a/comp/core/flare/providers.go +++ b/comp/core/flare/providers.go @@ -53,6 +53,7 @@ func (f *flare) collectLogsFiles(fb types.FlareBuilder) error { func (f *flare) collectConfigFiles(fb types.FlareBuilder) error { confSearchPaths := map[string]string{ "": f.config.GetString("confd_path"), + "fleet": filepath.Join(f.config.GetString("fleet_policies_dir"), "conf.d"), "dist": filepath.Join(f.params.distPath, "conf.d"), "checksd": f.params.pythonChecksPath, } diff --git a/comp/core/gui/guiimpl/checks.go b/comp/core/gui/guiimpl/checks.go index 058a75b08f05c..f54c682f72b8b 100644 --- a/comp/core/gui/guiimpl/checks.go +++ b/comp/core/gui/guiimpl/checks.go @@ -42,9 +42,19 @@ var ( filepath.Join(defaultpaths.GetDistPath(), "checks.d"), // Custom checks pkgconfigsetup.Datadog().GetString("additional_checksd"), // Custom checks defaultpaths.PyChecksPath, // Integrations-core checks + getFleetPoliciesPath(), // Fleet Policies } ) +// getFleetPoliciesPath returns the path to the fleet policies directory if it is set in the configuration +// otherwise it returns an empty string +func getFleetPoliciesPath() string { + if len(pkgconfigsetup.Datadog().GetString("fleet_policies_dir")) > 0 { + return filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d") + } + return "" +} + // Adds the specific handlers for /checks/ endpoints func checkHandler(r *mux.Router, collector collector.Component, ac autodiscovery.Component) { r.HandleFunc("/running", http.HandlerFunc(sendRunningChecks)).Methods("POST") @@ -208,6 +218,9 @@ func getCheckConfigFile(w http.ResponseWriter, r *http.Request) { var file []byte var e error for _, path := range configPaths { + if len(path) == 0 { + continue + } filePath, err := securejoin.SecureJoin(path, fileName) if err != nil { log.Errorf("Error: Unable to join config path with the file name: %s", fileName) @@ -443,7 +456,9 @@ func getConfigsInPath(path string) ([]string, error) { func listConfigs(w http.ResponseWriter, _ *http.Request) { filenames := []string{} for _, path := range configPaths { - + if len(path) == 0 { + continue + } configs, e := getConfigsInPath(path) if e != nil { log.Errorf("Unable to list configurations from %s: %v", path, e) diff --git a/comp/core/log/impl-trace/go.mod b/comp/core/log/impl-trace/go.mod index d78d19fde7f68..309b259ad3616 100644 --- a/comp/core/log/impl-trace/go.mod +++ b/comp/core/log/impl-trace/go.mod @@ -84,7 +84,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/comp/core/log/impl-trace/go.sum b/comp/core/log/impl-trace/go.sum index 9694f5d5e2355..dd16364891695 100644 --- a/comp/core/log/impl-trace/go.sum +++ b/comp/core/log/impl-trace/go.sum @@ -87,8 +87,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/core/log/impl/go.mod b/comp/core/log/impl/go.mod index 0d68073c13084..12d048a84bd4d 100644 --- a/comp/core/log/impl/go.mod +++ b/comp/core/log/impl/go.mod @@ -74,7 +74,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/comp/core/log/impl/go.sum b/comp/core/log/impl/go.sum index 9694f5d5e2355..dd16364891695 100644 --- a/comp/core/log/impl/go.sum +++ b/comp/core/log/impl/go.sum @@ -87,8 +87,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/core/log/mock/go.mod b/comp/core/log/mock/go.mod index 8b744f1e85ca2..e4f0f87cabdcb 100644 --- a/comp/core/log/mock/go.mod +++ b/comp/core/log/mock/go.mod @@ -58,6 +58,7 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/config/structure => ../../../../pkg/config/structure diff --git a/comp/core/log/mock/go.sum b/comp/core/log/mock/go.sum index 2ad11aee593c1..4b0aad4f95926 100644 --- a/comp/core/log/mock/go.sum +++ b/comp/core/log/mock/go.sum @@ -80,8 +80,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/comp/core/status/render_helpers.go b/comp/core/status/render_helpers.go index 7ee0830c50e1b..4c4769927d4c9 100644 --- a/comp/core/status/render_helpers.go +++ b/comp/core/status/render_helpers.go @@ -23,8 +23,6 @@ import ( "github.com/fatih/color" "github.com/spf13/cast" - "golang.org/x/text/cases" - "golang.org/x/text/language" "golang.org/x/text/unicode/norm" ) @@ -290,8 +288,12 @@ func formatTitle(title string) string { } title = strings.Join(words, " ") + if title == "" { + return "" + } + // Capitalize the first letter - return cases.Title(language.English, cases.NoLower).String(title) + return strings.ToUpper(string(title[0])) + title[1:] } func status(check map[string]interface{}) string { diff --git a/comp/core/status/statusimpl/go.mod b/comp/core/status/statusimpl/go.mod index 9487310619057..c121259bfa141 100644 --- a/comp/core/status/statusimpl/go.mod +++ b/comp/core/status/statusimpl/go.mod @@ -88,7 +88,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/comp/core/status/statusimpl/go.sum b/comp/core/status/statusimpl/go.sum index fafe0837efa7e..73fea28e04146 100644 --- a/comp/core/status/statusimpl/go.sum +++ b/comp/core/status/statusimpl/go.sum @@ -93,8 +93,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/core/tagger/collectors/pod_tag_extractor.go b/comp/core/tagger/collectors/pod_tag_extractor.go index 669a71584f609..aded502edaf4c 100644 --- a/comp/core/tagger/collectors/pod_tag_extractor.go +++ b/comp/core/tagger/collectors/pod_tag_extractor.go @@ -34,6 +34,8 @@ func (p *PodTagExtractor) Extract(podEntity *workloadmeta.KubernetesPod, cardina return append(tagInfos.LowCardTags, tagInfos.OrchestratorCardTags...) case types.LowCardinality: return tagInfos.LowCardTags + case types.NoneCardinality: + return []string{} default: log.Errorf("unsupported tag cardinality %v", cardinality) return []string{} diff --git a/comp/core/tagger/def/component.go b/comp/core/tagger/def/component.go index c580325c3bf81..619196f41de95 100644 --- a/comp/core/tagger/def/component.go +++ b/comp/core/tagger/def/component.go @@ -9,6 +9,7 @@ package tagger import ( "context" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger/types" taggertypes "github.com/DataDog/datadog-agent/pkg/tagger/types" @@ -37,6 +38,7 @@ type Component interface { // integrations using the tagger LegacyTag(entity string, cardinality types.TagCardinality) ([]string, error) Tag(entityID types.EntityID, cardinality types.TagCardinality) ([]string, error) + GenerateContainerIDFromOriginInfo(originInfo origindetection.OriginInfo) (string, error) AccumulateTagsFor(entityID types.EntityID, cardinality types.TagCardinality, tb tagset.TagsAccumulator) error Standard(entityID types.EntityID) ([]string, error) List() types.TaggerListResponse diff --git a/comp/core/tagger/impl-noop/tagger.go b/comp/core/tagger/impl-noop/tagger.go index 811a60e905923..986e113df2c1e 100644 --- a/comp/core/tagger/impl-noop/tagger.go +++ b/comp/core/tagger/impl-noop/tagger.go @@ -17,6 +17,7 @@ import ( "context" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger/types" taggertypes "github.com/DataDog/datadog-agent/pkg/tagger/types" @@ -49,6 +50,12 @@ func (n *noopTagger) LegacyTag(string, types.TagCardinality) ([]string, error) { return nil, nil } +// GenerateContainerIDFromOriginInfo generates a container ID from Origin Info. +// This is a no-op for the noop tagger +func (n *noopTagger) GenerateContainerIDFromOriginInfo(origindetection.OriginInfo) (string, error) { + return "", nil +} + func (n *noopTagger) AccumulateTagsFor(types.EntityID, types.TagCardinality, tagset.TagsAccumulator) error { return nil } diff --git a/comp/core/tagger/impl-remote/remote.go b/comp/core/tagger/impl-remote/remote.go index b1a77165f18a9..b602facbb824b 100644 --- a/comp/core/tagger/impl-remote/remote.go +++ b/comp/core/tagger/impl-remote/remote.go @@ -27,6 +27,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/core/tagger/utils" @@ -47,6 +48,8 @@ const ( var errTaggerStreamNotStarted = errors.New("tagger stream not started") +var errTaggerFailedGenerateContainerIDFromOriginInfo = errors.New("tagger failed to generate container ID from origin info") + // Requires defines the dependencies for the remote tagger. type Requires struct { compdef.In @@ -75,6 +78,7 @@ type remoteTagger struct { log log.Component conn *grpc.ClientConn + token string client pb.AgentSecureClient stream pb.AgentSecure_TaggerStreamEntitiesClient @@ -82,6 +86,9 @@ type remoteTagger struct { streamCancel context.CancelFunc filter *types.Filter + queryCtx context.Context + queryCancel context.CancelFunc + ctx context.Context cancel context.CancelFunc @@ -250,6 +257,79 @@ func (t *remoteTagger) LegacyTag(entity string, cardinality types.TagCardinality return t.Tag(entityID, cardinality) } +// GenerateContainerIDFromOriginInfo returns a container ID for the given Origin Info. +// This function currently only uses the External Data from the Origin Info to generate the container ID. +func (t *remoteTagger) GenerateContainerIDFromOriginInfo(originInfo origindetection.OriginInfo) (string, error) { + fail := true + defer func() { + if fail { + t.telemetryStore.OriginInfoRequests.Inc("failed") + } else { + t.telemetryStore.OriginInfoRequests.Inc("success") + } + }() + + expBackoff := backoff.NewExponentialBackOff() + expBackoff.InitialInterval = 500 * time.Millisecond + expBackoff.MaxInterval = 1 * time.Second + expBackoff.MaxElapsedTime = 15 * time.Second + + var containerID string + + err := backoff.Retry(func() error { + select { + case <-t.ctx.Done(): + return &backoff.PermanentError{Err: errTaggerFailedGenerateContainerIDFromOriginInfo} + default: + } + + // Fetch the auth token + if t.token == "" { + var authError error + t.token, authError = t.options.TokenFetcher() + if authError != nil { + _ = t.log.Errorf("unable to fetch auth token, will possibly retry: %s", authError) + return authError + } + } + + // Create the context with the auth token + t.queryCtx, t.queryCancel = context.WithCancel( + metadata.NewOutgoingContext(t.ctx, metadata.MD{ + "authorization": []string{fmt.Sprintf("Bearer %s", t.token)}, + }), + ) + + // Call the gRPC method to get the container ID from the origin info + containerIDResponse, err := t.client.TaggerGenerateContainerIDFromOriginInfo(t.queryCtx, &pb.GenerateContainerIDFromOriginInfoRequest{ + ExternalData: &pb.GenerateContainerIDFromOriginInfoRequest_ExternalData{ + Init: &originInfo.ExternalData.Init, + ContainerName: &originInfo.ExternalData.ContainerName, + PodUID: &originInfo.ExternalData.PodUID, + }, + }) + if err != nil { + _ = t.log.Errorf("unable to generate container ID from origin info, will retry: %s", err) + return err + } + + if containerIDResponse == nil { + _ = t.log.Warnf("unable to generate container ID from origin info, will retry: %s", err) + return errors.New("containerIDResponse is nil") + } + containerID = containerIDResponse.ContainerID + + fail = false + t.log.Debugf("Container ID generated successfully from origin info %+v: %s", originInfo, containerID) + return nil + }, expBackoff) + + if err != nil { + return "", err + } + return containerID, nil +} + // AccumulateTagsFor returns tags for a given entity at the desired cardinality. func (t *remoteTagger) AccumulateTagsFor(entityID types.EntityID, cardinality types.TagCardinality, tb tagset.TagsAccumulator) error { tags, err := t.Tag(entityID, cardinality) diff --git a/comp/core/tagger/impl/local_tagger.go b/comp/core/tagger/impl/local_tagger.go index ac2259342eb25..85c8741c52b02 100644 --- a/comp/core/tagger/impl/local_tagger.go +++ b/comp/core/tagger/impl/local_tagger.go @@ -9,16 +9,20 @@ import ( "context" "fmt" "sync" + "time" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/tagger/collectors" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/tagstore" "github.com/DataDog/datadog-agent/comp/core/tagger/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger/types" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" taggertypes "github.com/DataDog/datadog-agent/pkg/tagger/types" "github.com/DataDog/datadog-agent/pkg/tagset" + "github.com/DataDog/datadog-agent/pkg/util/containers/metrics" + "github.com/DataDog/datadog-agent/pkg/util/optional" ) // Tagger is the entry class for entity tagging. It hold the tagger collector, @@ -99,6 +103,12 @@ func (t *localTagger) Tag(entityID types.EntityID, cardinality types.TagCardinal return tags.Copy(), nil } +// GenerateContainerIDFromOriginInfo generates a container ID from Origin Info. +func (t *localTagger) GenerateContainerIDFromOriginInfo(originInfo origindetection.OriginInfo) (string, error) { + metaCollector := metrics.GetProvider(optional.NewOption(t.workloadStore)).GetMetaCollector() + return metaCollector.ContainerIDForPodUIDAndContName(originInfo.ExternalData.PodUID, originInfo.ExternalData.ContainerName, originInfo.ExternalData.Init, time.Second) +} + // LegacyTag has the same behaviour as the Tag method, but it receives the entity id as a string and parses it. // If possible, avoid using this function, and use the Tag method instead. // This function exists in order not to break backward compatibility with rtloader and python diff --git a/comp/core/tagger/impl/replay_tagger.go b/comp/core/tagger/impl/replay_tagger.go index ddc9851a898db..6f012e9c9d680 100644 --- a/comp/core/tagger/impl/replay_tagger.go +++ b/comp/core/tagger/impl/replay_tagger.go @@ -11,6 +11,7 @@ import ( "time" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/tagstore" "github.com/DataDog/datadog-agent/comp/core/tagger/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger/types" @@ -77,6 +78,12 @@ func (t *replayTagger) LegacyTag(entity string, cardinality types.TagCardinality return t.Tag(entityID, cardinality) } +// GenerateContainerIDFromOriginInfo generates a container ID from Origin Info. +// This is a no-op for the replay tagger +func (t *replayTagger) GenerateContainerIDFromOriginInfo(origindetection.OriginInfo) (string, error) { + return "", nil +} + // AccumulateTagsFor returns tags for a given entity at the desired cardinality. func (t *replayTagger) AccumulateTagsFor(entityID types.EntityID, cardinality types.TagCardinality, tb tagset.TagsAccumulator) error { tags := t.store.LookupHashed(entityID, cardinality) diff --git a/comp/core/tagger/impl/tagger.go b/comp/core/tagger/impl/tagger.go index 1ef1a137d6a34..de05b9f6eaa09 100644 --- a/comp/core/tagger/impl/tagger.go +++ b/comp/core/tagger/impl/tagger.go @@ -27,6 +27,7 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" taggermock "github.com/DataDog/datadog-agent/comp/core/tagger/mock" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/core/tagger/utils" @@ -421,7 +422,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert // | none | empty || empty | // | empty | not empty || container prefix + originFromMsg | // | none | not empty || container prefix + originFromMsg | - if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == "none" { + if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" @@ -459,8 +460,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert } default: // Disable origin detection if cardinality is none - // TODO: The `none` cardinality should be directly supported by the Tagger. - if originInfo.Cardinality == "none" { + if originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" @@ -538,7 +538,12 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert } } -// generateContainerIDFromExternalData generates a container ID from the external data +// GenerateContainerIDFromOriginInfo generates a container ID from Origin Info. +func (t *TaggerWrapper) GenerateContainerIDFromOriginInfo(originInfo origindetection.OriginInfo) (string, error) { + return t.defaultTagger.GenerateContainerIDFromOriginInfo(originInfo) +} + +// generateContainerIDFromExternalData generates a container ID from the External Data. func (t *TaggerWrapper) generateContainerIDFromExternalData(e externalData, metricsProvider provider.ContainerIDForPodUIDAndContNameRetriever) (string, error) { return metricsProvider.ContainerIDForPodUIDAndContName(e.podUID, e.containerName, e.init, time.Second) } diff --git a/comp/core/tagger/mock/fake_tagger.go b/comp/core/tagger/mock/fake_tagger.go index b4ff66a65d2c6..8a609d052c766 100644 --- a/comp/core/tagger/mock/fake_tagger.go +++ b/comp/core/tagger/mock/fake_tagger.go @@ -10,6 +10,7 @@ import ( "strconv" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/tagstore" "github.com/DataDog/datadog-agent/comp/core/tagger/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger/types" @@ -118,6 +119,11 @@ func (f *FakeTagger) LegacyTag(entity string, cardinality types.TagCardinality) return f.Tag(entityID, cardinality) } +// GenerateContainerIDFromOriginInfo fake implementation +func (f *FakeTagger) GenerateContainerIDFromOriginInfo(origindetection.OriginInfo) (string, error) { + return "", nil +} + // GlobalTags fake implementation func (f *FakeTagger) GlobalTags(cardinality types.TagCardinality) ([]string, error) { return f.Tag(types.GetGlobalEntityID(), cardinality) diff --git a/comp/core/tagger/origindetection/go.mod b/comp/core/tagger/origindetection/go.mod new file mode 100644 index 0000000000000..10e53c86ea39d --- /dev/null +++ b/comp/core/tagger/origindetection/go.mod @@ -0,0 +1,14 @@ +module github.com/DataDog/datadog-agent/comp/core/tagger/origindetection + +go 1.22.0 + +require github.com/stretchr/testify v1.10.0 + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/comp/core/tagger/origindetection/go.sum b/comp/core/tagger/origindetection/go.sum new file mode 100644 index 0000000000000..cec386e1e2769 --- /dev/null +++ b/comp/core/tagger/origindetection/go.sum @@ -0,0 +1,23 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/comp/core/tagger/origindetection/origindetection.go b/comp/core/tagger/origindetection/origindetection.go new file mode 100644 index 0000000000000..712792c54f298 --- /dev/null +++ b/comp/core/tagger/origindetection/origindetection.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// TODO: A lot of the code in this file is currently duplicated in taggertypes. +// We will need to move all the code in taggertype to this file and remove the taggertypes package. + +// Package origindetection contains the types and functions used for Origin Detection. +package origindetection + +import ( + "strconv" + "strings" +) + +// ProductOrigin is the origin of the product that sent the entity. +type ProductOrigin int + +const ( + // ProductOriginDogStatsDLegacy is the ProductOrigin for DogStatsD in Legacy mode. + // TODO: remove this when dogstatsd_origin_detection_unified is enabled by default + ProductOriginDogStatsDLegacy ProductOrigin = iota + // ProductOriginDogStatsD is the ProductOrigin for DogStatsD. + ProductOriginDogStatsD ProductOrigin = iota + // ProductOriginAPM is the ProductOrigin for APM. + ProductOriginAPM ProductOrigin = iota + + // External Data Prefixes + // These prefixes are used to build the External Data Environment Variable. + + // ExternalDataInitPrefix is the prefix for the Init flag in the External Data. + ExternalDataInitPrefix = "it-" + // ExternalDataContainerNamePrefix is the prefix for the Container Name in the External Data. + ExternalDataContainerNamePrefix = "cn-" + // ExternalDataPodUIDPrefix is the prefix for the Pod UID in the External Data. + ExternalDataPodUIDPrefix = "pu-" +) + +// OriginInfo contains the Origin Detection information. +type OriginInfo struct { + LocalData LocalData // LocalData is the local data list. + ExternalData ExternalData // ExternalData is the external data list. + Cardinality string // Cardinality is the cardinality of the resolved origin. + ProductOrigin ProductOrigin // ProductOrigin is the product that sent the origin information. +} + +// LocalData that is generated by the client and sent to the Agent. +type LocalData struct { + ProcessID uint32 // ProcessID of the container process on the host. + ContainerID string // ContainerID sent from the client. + Inode uint64 // Inode is the Cgroup inode of the container. + PodUID string // PodUID of the pod sent from the client. +} + +// ExternalData generated by the Admission Controller and sent to the Agent. +type ExternalData struct { + Init bool // Init is true if the container is an init container. + ContainerName string // ContainerName is the name of the container as seen by the Admission Controller. + PodUID string // PodUID is the UID of the pod as seen by the Admission Controller. +} + +// GenerateContainerIDFromExternalData generates a container ID from the external data. +type GenerateContainerIDFromExternalData func(externalData ExternalData) (string, error) + +// ParseExternalData parses the external data string into an ExternalData struct. +func ParseExternalData(externalEnv string) (ExternalData, error) { + if externalEnv == "" { + return ExternalData{}, nil + } + var externalData ExternalData + var parsingError error + for _, item := range strings.Split(externalEnv, ",") { + switch { + case strings.HasPrefix(item, ExternalDataInitPrefix): + externalData.Init, parsingError = strconv.ParseBool(item[len(ExternalDataInitPrefix):]) + case strings.HasPrefix(item, ExternalDataContainerNamePrefix): + externalData.ContainerName = item[len(ExternalDataContainerNamePrefix):] + case strings.HasPrefix(item, ExternalDataPodUIDPrefix): + externalData.PodUID = item[len(ExternalDataPodUIDPrefix):] + } + } + return externalData, parsingError +} diff --git a/comp/core/tagger/origindetection/origindetection_test.go b/comp/core/tagger/origindetection/origindetection_test.go new file mode 100644 index 0000000000000..a873093f6191b --- /dev/null +++ b/comp/core/tagger/origindetection/origindetection_test.go @@ -0,0 +1,81 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package origindetection contains the types and functions used for Origin Detection. +package origindetection + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseExternalData(t *testing.T) { + tests := []struct { + name string + externalEnv string + expectedData ExternalData + expectedError bool + }{ + { + name: "Empty external data", + externalEnv: "", + expectedData: ExternalData{ + Init: false, + ContainerName: "", + PodUID: "", + }, + expectedError: false, + }, + { + name: "Valid external data with Init", + externalEnv: "it-true,cn-container-name,pu-12345678-90ab-cdef-1234-567890abcdef", + expectedData: ExternalData{ + Init: true, + ContainerName: "container-name", + PodUID: "12345678-90ab-cdef-1234-567890abcdef", + }, + expectedError: false, + }, + { + name: "Invalid Init value", + externalEnv: "it-invalid,cn-container-name,pu-12345678-90ab-cdef-1234-567890abcdef", + expectedData: ExternalData{}, + expectedError: true, + }, + { + name: "Partial external data", + externalEnv: "cn-container-name", + expectedData: ExternalData{ + Init: false, + ContainerName: "container-name", + PodUID: "", + }, + expectedError: false, + }, + { + name: "Unrecognized prefix", + externalEnv: "unknown-prefix", + expectedData: ExternalData{ + Init: false, + ContainerName: "", + PodUID: "", + }, + expectedError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result, err := ParseExternalData(tc.externalEnv) + + if tc.expectedError { + assert.Error(t, err) + } else { + assert.Equal(t, tc.expectedData, result) + } + }) + } +} diff --git a/comp/core/tagger/server/server.go b/comp/core/tagger/server/server.go index 5e323c43c7ee2..34a719f32c440 100644 --- a/comp/core/tagger/server/server.go +++ b/comp/core/tagger/server/server.go @@ -17,6 +17,7 @@ import ( "github.com/google/uuid" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/proto" "github.com/DataDog/datadog-agent/comp/core/tagger/types" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" @@ -165,3 +166,22 @@ func (s *Server) TaggerFetchEntity(_ context.Context, in *pb.FetchEntityRequest) Tags: tags, }, nil } + +// TaggerGenerateContainerIDFromOriginInfo request the generation of a container ID from external data from the Tagger. +// This function takes an Origin Info but only uses the ExternalData part of it, this is done for backward compatibility. +func (s *Server) TaggerGenerateContainerIDFromOriginInfo(_ context.Context, in *pb.GenerateContainerIDFromOriginInfoRequest) (*pb.GenerateContainerIDFromOriginInfoResponse, error) { + generatedContainerID, err := s.taggerComponent.GenerateContainerIDFromOriginInfo(origindetection.OriginInfo{ + ExternalData: origindetection.ExternalData{ + Init: *in.ExternalData.Init, + ContainerName: *in.ExternalData.ContainerName, + PodUID: *in.ExternalData.PodUID, + }, + }) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "%s", err) + } + + return &pb.GenerateContainerIDFromOriginInfoResponse{ + ContainerID: generatedContainerID, + }, nil +} diff --git a/comp/core/tagger/subscriber/subscription_manager_test.go b/comp/core/tagger/subscriber/subscription_manager_test.go index b9684af30e2d4..dbebd8ed23d98 100644 --- a/comp/core/tagger/subscriber/subscription_manager_test.go +++ b/comp/core/tagger/subscriber/subscription_manager_test.go @@ -109,6 +109,21 @@ func TestSubscriptionManager(t *testing.T) { highCardSubscription.Unsubscribe() + // None Cardinality Subscriber + noneCardSubID := "none-card-sub" + noneCardSubscription, err := sm.Subscribe(noneCardSubID, types.NewFilterBuilder().Include(types.EntityIDPrefix("foo")).Build(types.NoneCardinality), nil) + require.NoError(t, err) + + sm.Notify([]types.EntityEvent{ + events["added"], + events["modified"], + events["deleted"], + events["added-with-no-id"], + events["added-with-unmatched-prefix"], + }) + + noneCardSubscription.Unsubscribe() + // Verify low cardinality subscriber received events assertReceivedEvents(t, lowCardSubscription.EventsChan(), []types.EntityEvent{ { @@ -192,6 +207,28 @@ func TestSubscriptionManager(t *testing.T) { }, }, }) + + // Verify none cardinality subscriber received events + assertReceivedEvents(t, noneCardSubscription.EventsChan(), []types.EntityEvent{ + { + EventType: types.EventTypeAdded, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeModified, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeDeleted, + Entity: types.Entity{ + ID: entityID, + }, + }, + }) } func assertReceivedEvents(t *testing.T, ch chan []types.EntityEvent, expectedEvents []types.EntityEvent) { diff --git a/comp/core/tagger/tagstore/entity_tags.go b/comp/core/tagger/tagstore/entity_tags.go index 3488a8e345c16..ed62d45849109 100644 --- a/comp/core/tagger/tagstore/entity_tags.go +++ b/comp/core/tagger/tagstore/entity_tags.go @@ -109,12 +109,16 @@ func (e *EntityTagsWithMultipleSources) getStandard() []string { func (e *EntityTagsWithMultipleSources) getHashedTags(cardinality types.TagCardinality) tagset.HashedTags { e.computeCache() - if cardinality == types.HighCardinality { + switch cardinality { + case types.HighCardinality: return e.cachedAll - } else if cardinality == types.OrchestratorCardinality { + case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} + default: + return e.cachedLow } - return e.cachedLow } func (e *EntityTagsWithMultipleSources) computeCache() { @@ -302,6 +306,8 @@ func (e *EntityTagsWithSingleSource) getHashedTags(cardinality types.TagCardinal return e.cachedAll case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} default: return e.cachedLow } diff --git a/comp/core/tagger/tagstore/entity_tags_test.go b/comp/core/tagger/tagstore/entity_tags_test.go index c93d764377aaf..16e2d4b077f86 100644 --- a/comp/core/tagger/tagstore/entity_tags_test.go +++ b/comp/core/tagger/tagstore/entity_tags_test.go @@ -100,6 +100,12 @@ func TestGetHashedTags(t *testing.T) { []string{"l1:v1", "l2:v2", "service:s1", "o1:v1", "o2:v2", "h1:v1", "h2:v2"}, entityTags.getHashedTags(types.HighCardinality).Get(), ) + + assert.Equal( + t, + []string(nil), + entityTags.getHashedTags(types.NoneCardinality).Get(), + ) } func TestTagsForSource(t *testing.T) { diff --git a/comp/core/tagger/tagstore/tagstore_test.go b/comp/core/tagger/tagstore/tagstore_test.go index 4699aa6ad56cb..0d992b682ca68 100644 --- a/comp/core/tagger/tagstore/tagstore_test.go +++ b/comp/core/tagger/tagstore/tagstore_test.go @@ -88,10 +88,12 @@ func (s *StoreTestSuite) TestLookup() { tagsHigh := s.tagstore.Lookup(entityID, types.HighCardinality) tagsOrch := s.tagstore.Lookup(entityID, types.OrchestratorCardinality) tagsLow := s.tagstore.Lookup(entityID, types.LowCardinality) + tagsNone := s.tagstore.Lookup(entityID, types.NoneCardinality) assert.Len(s.T(), tagsHigh, 4) assert.Len(s.T(), tagsLow, 2) assert.Len(s.T(), tagsOrch, 3) + assert.Nil(s.T(), tagsNone) } func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { @@ -118,10 +120,12 @@ func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { tagsLow := s.tagstore.LookupHashedWithEntityStr(entityID, types.LowCardinality) tagsOrch := s.tagstore.LookupHashedWithEntityStr(entityID, types.OrchestratorCardinality) tagsHigh := s.tagstore.LookupHashedWithEntityStr(entityID, types.HighCardinality) + tagsNone := s.tagstore.LookupHashedWithEntityStr(entityID, types.NoneCardinality) assert.ElementsMatch(s.T(), tagsLow.Get(), []string{"low1", "low2"}) assert.ElementsMatch(s.T(), tagsOrch.Get(), []string{"low1", "low2", "orchestrator1"}) assert.ElementsMatch(s.T(), tagsHigh.Get(), []string{"low1", "low2", "orchestrator1", "high1"}) + assert.ElementsMatch(s.T(), tagsNone.Get(), []string{}) } func (s *StoreTestSuite) TestLookupStandard() { diff --git a/comp/core/tagger/telemetry/telemetry.go b/comp/core/tagger/telemetry/telemetry.go index 8b50202dd2e99..0682475eb6d8e 100644 --- a/comp/core/tagger/telemetry/telemetry.go +++ b/comp/core/tagger/telemetry/telemetry.go @@ -61,9 +61,14 @@ type Store struct { // notification with a group of events. Receives telemetry.Counter + // OriginInfoRequests tracks the number of requests to the Tagger + // to generate a container ID from Origin Info. + OriginInfoRequests telemetry.Counter + LowCardinalityQueries CardinalityTelemetry OrchestratorCardinalityQueries CardinalityTelemetry HighCardinalityQueries CardinalityTelemetry + NoneCardinalityQueries CardinalityTelemetry UnknownCardinalityQueries CardinalityTelemetry } @@ -131,9 +136,16 @@ func NewStore(telemetryComp telemetry.Component) *Store { []string{}, "Number of of times the tagger has received a notification with a group of events", telemetry.Options{NoDoubleUnderscoreSep: true}), + // OriginInfoRequests tracks the number of requests to the tagger + // to generate a container ID from origin info. + OriginInfoRequests: telemetryComp.NewCounterWithOpts(subsystem, "origin_info_requests", + []string{"status"}, "Number of requests to the tagger to generate a container ID from origin info.", + telemetry.Options{NoDoubleUnderscoreSep: true}), + LowCardinalityQueries: newCardinalityTelemetry(queries, types.LowCardinalityString), OrchestratorCardinalityQueries: newCardinalityTelemetry(queries, types.OrchestratorCardinalityString), HighCardinalityQueries: newCardinalityTelemetry(queries, types.HighCardinalityString), + NoneCardinalityQueries: newCardinalityTelemetry(queries, types.NoneCardinalityString), UnknownCardinalityQueries: newCardinalityTelemetry(queries, types.UnknownCardinalityString), } }) @@ -150,6 +162,8 @@ func (s *Store) QueriesByCardinality(card types.TagCardinality) *CardinalityTele return &s.OrchestratorCardinalityQueries case types.HighCardinality: return &s.HighCardinalityQueries + case types.NoneCardinality: + return &s.NoneCardinalityQueries default: return &s.UnknownCardinalityQueries } diff --git a/comp/core/tagger/types/types.go b/comp/core/tagger/types/types.go index e45f9e6daa084..dc53af7baf27b 100644 --- a/comp/core/tagger/types/types.go +++ b/comp/core/tagger/types/types.go @@ -78,6 +78,7 @@ const ( LowCardinality TagCardinality = iota OrchestratorCardinality HighCardinality + NoneCardinality ) // Entity is an entity ID + tags. @@ -92,6 +93,10 @@ type Entity struct { // GetTags flattens all tags from all cardinalities into a single slice of tag // strings. func (e Entity) GetTags(cardinality TagCardinality) []string { + if cardinality == NoneCardinality { + return []string{} + } + tagArrays := make([][]string, 0, 3) tagArrays = append(tagArrays, e.LowCardinalityTags) @@ -117,6 +122,11 @@ func (e Entity) Copy(cardinality TagCardinality) Entity { case LowCardinality: newEntity.HighCardinalityTags = nil newEntity.OrchestratorCardinalityTags = nil + case NoneCardinality: + newEntity.HighCardinalityTags = nil + newEntity.OrchestratorCardinalityTags = nil + newEntity.LowCardinalityTags = nil + newEntity.StandardTags = nil } return newEntity @@ -131,6 +141,8 @@ const ( ShortOrchestratorCardinalityString = "orch" // HighCardinalityString is the string representation of the high cardinality HighCardinalityString = "high" + // NoneCardinalityString is the string representation of the none cardinality + NoneCardinalityString = "none" // UnknownCardinalityString represents an unknown level of cardinality UnknownCardinalityString = "unknown" ) @@ -145,6 +157,8 @@ func StringToTagCardinality(c string) (TagCardinality, error) { return OrchestratorCardinality, nil case LowCardinalityString: return LowCardinality, nil + case NoneCardinalityString: + return NoneCardinality, nil default: return LowCardinality, fmt.Errorf("unsupported value %s received for tag cardinality", c) } @@ -160,6 +174,8 @@ func TagCardinalityToString(c TagCardinality) string { return OrchestratorCardinalityString case LowCardinality: return LowCardinalityString + case NoneCardinality: + return NoneCardinalityString default: return UnknownCardinalityString } diff --git a/comp/forwarder/defaultforwarder/go.mod b/comp/forwarder/defaultforwarder/go.mod index a0891b10fda75..26fee04e02f54 100644 --- a/comp/forwarder/defaultforwarder/go.mod +++ b/comp/forwarder/defaultforwarder/go.mod @@ -78,7 +78,6 @@ require ( github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 go.uber.org/fx v1.23.0 - golang.org/x/text v0.21.0 ) require ( @@ -113,7 +112,7 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -146,8 +145,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/comp/forwarder/defaultforwarder/go.sum b/comp/forwarder/defaultforwarder/go.sum index 2e4c8055f6ab0..75d47f4e88e64 100644 --- a/comp/forwarder/defaultforwarder/go.sum +++ b/comp/forwarder/defaultforwarder/go.sum @@ -97,8 +97,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -281,8 +281,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/forwarder/defaultforwarder/internal/retry/telemetry.go b/comp/forwarder/defaultforwarder/internal/retry/telemetry.go index ba3d2c7c35f53..f676c5bc75d8a 100644 --- a/comp/forwarder/defaultforwarder/internal/retry/telemetry.go +++ b/comp/forwarder/defaultforwarder/internal/retry/telemetry.go @@ -9,9 +9,6 @@ import ( "expvar" "strings" - "golang.org/x/text/cases" - "golang.org/x/text/language" - "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" "github.com/DataDog/datadog-agent/pkg/telemetry" ) @@ -308,11 +305,14 @@ func (t onDiskRetryQueueTelemetry) addDeserializeTransactionsCount(count int) { } func toCamelCase(s string) string { - caser := cases.Title(language.English) parts := strings.Split(s, "_") var camelCase string for _, p := range parts { - camelCase += caser.String(p) + if p == "" { + continue + } + camelCase += strings.ToUpper(string(p[0])) + camelCase += string(p[1:]) } return camelCase } diff --git a/comp/forwarder/orchestrator/orchestratorinterface/go.mod b/comp/forwarder/orchestrator/orchestratorinterface/go.mod index 3022f485e02a3..4fc6e333ce46b 100644 --- a/comp/forwarder/orchestrator/orchestratorinterface/go.mod +++ b/comp/forwarder/orchestrator/orchestratorinterface/go.mod @@ -117,7 +117,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -153,7 +153,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/forwarder/orchestrator/orchestratorinterface/go.sum b/comp/forwarder/orchestrator/orchestratorinterface/go.sum index 68e574e5c8151..a055847bfd129 100644 --- a/comp/forwarder/orchestrator/orchestratorinterface/go.sum +++ b/comp/forwarder/orchestrator/orchestratorinterface/go.sum @@ -100,8 +100,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/logs/agent/config/go.mod b/comp/logs/agent/config/go.mod index 5e99aa77604f0..1ae81a7be7da6 100644 --- a/comp/logs/agent/config/go.mod +++ b/comp/logs/agent/config/go.mod @@ -76,7 +76,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/comp/logs/agent/config/go.sum b/comp/logs/agent/config/go.sum index 9694f5d5e2355..dd16364891695 100644 --- a/comp/logs/agent/config/go.sum +++ b/comp/logs/agent/config/go.sum @@ -87,8 +87,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/otelcol/collector-contrib/def/go.mod b/comp/otelcol/collector-contrib/def/go.mod index b4b92cf6fe357..de371feafc0c9 100644 --- a/comp/otelcol/collector-contrib/def/go.mod +++ b/comp/otelcol/collector-contrib/def/go.mod @@ -107,9 +107,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect - golang.org/x/net v0.31.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect gonum.org/v1/gonum v0.15.1 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/collector-contrib/def/go.sum b/comp/otelcol/collector-contrib/def/go.sum index b0340b1c9c43d..258b336c23443 100644 --- a/comp/otelcol/collector-contrib/def/go.sum +++ b/comp/otelcol/collector-contrib/def/go.sum @@ -265,8 +265,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -278,12 +278,12 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/comp/otelcol/collector-contrib/impl/go.mod b/comp/otelcol/collector-contrib/impl/go.mod index e816b6e25e500..9b14be986534c 100644 --- a/comp/otelcol/collector-contrib/impl/go.mod +++ b/comp/otelcol/collector-contrib/impl/go.mod @@ -312,15 +312,15 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.29.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.9.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/term v0.26.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.27.0 // indirect gonum.org/v1/gonum v0.15.1 // indirect diff --git a/comp/otelcol/collector-contrib/impl/go.sum b/comp/otelcol/collector-contrib/impl/go.sum index 5672b3b4c0c95..c97ea3c785b72 100644 --- a/comp/otelcol/collector-contrib/impl/go.sum +++ b/comp/otelcol/collector-contrib/impl/go.sum @@ -1120,8 +1120,8 @@ golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1218,8 +1218,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1241,8 +1241,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1319,8 +1319,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1334,8 +1334,8 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1352,8 +1352,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/comp/otelcol/converter/impl/go.mod b/comp/otelcol/converter/impl/go.mod index b3bae2ce5b62b..ec9c75f1a3c68 100644 --- a/comp/otelcol/converter/impl/go.mod +++ b/comp/otelcol/converter/impl/go.mod @@ -86,7 +86,7 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect diff --git a/comp/otelcol/converter/impl/go.sum b/comp/otelcol/converter/impl/go.sum index a98a26b75f480..4239a8ba70c9d 100644 --- a/comp/otelcol/converter/impl/go.sum +++ b/comp/otelcol/converter/impl/go.sum @@ -89,8 +89,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/otelcol/ddflareextension/def/go.mod b/comp/otelcol/ddflareextension/def/go.mod index 7ac807821aee4..2160a62e2f369 100644 --- a/comp/otelcol/ddflareextension/def/go.mod +++ b/comp/otelcol/ddflareextension/def/go.mod @@ -16,7 +16,7 @@ require ( go.opentelemetry.io/otel/trace v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/ddflareextension/def/go.sum b/comp/otelcol/ddflareextension/def/go.sum index cb77fcd0e24d3..e8348caae0f44 100644 --- a/comp/otelcol/ddflareextension/def/go.sum +++ b/comp/otelcol/ddflareextension/def/go.sum @@ -45,8 +45,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 0719700f9222a..25448ad26a8c3 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -13,6 +13,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/tagger/tags => ../../../core/tagger/tags github.com/DataDog/datadog-agent/comp/core/tagger/types => ../../../core/tagger/types github.com/DataDog/datadog-agent/comp/core/tagger/utils => ../../../core/tagger/utils @@ -151,6 +152,7 @@ require ( require go.opentelemetry.io/collector/extension/extensiontest v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect @@ -358,7 +360,7 @@ require ( github.com/hashicorp/hcl v1.0.1-vault-5 // indirect github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 // indirect github.com/hashicorp/serf v0.10.1 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/hetznercloud/hcloud-go/v2 v2.10.2 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -505,7 +507,7 @@ require ( golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index 7a6bcb0649c64..1b8a5abd738cb 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -443,8 +443,8 @@ github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 h1:fgVfQ4AC1av github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/hetznercloud/hcloud-go/v2 v2.10.2 h1:9gyTUPhfNbfbS40Spgij5mV5k37bOZgt8iHKCbfGs5I= github.com/hetznercloud/hcloud-go/v2 v2.10.2/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -1139,8 +1139,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/comp/otelcol/ddflareextension/impl/server.go b/comp/otelcol/ddflareextension/impl/server.go index ce1d2ceadef52..233dcddee1977 100644 --- a/comp/otelcol/ddflareextension/impl/server.go +++ b/comp/otelcol/ddflareextension/impl/server.go @@ -116,7 +116,7 @@ func newServer(endpoint string, handler http.Handler, auth bool) (*server, error // no easy way currently to pass required bearer auth token to OSS collector; // skip the validation if running inside a separate collector // TODO: determine way to allow OSS collector to authenticate with agent, OTEL-2226 - if auth { + if auth && util.GetAuthToken() != "" { r.Use(validateToken) } diff --git a/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-enhanced-result.yaml b/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-enhanced-result.yaml index 5914d58d690a1..055f7c67144eb 100644 --- a/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-enhanced-result.yaml +++ b/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-enhanced-result.yaml @@ -88,7 +88,7 @@ exporters: peer_service_aggregation: false peer_tags: [] peer_tags_aggregation: false - span_name_as_resource_name: true + span_name_as_resource_name: false span_name_remappings: {} trace_buffer: 0 write_buffer_size: 0 diff --git a/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-provided-result.yaml b/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-provided-result.yaml index 807d80682416a..d0b9a8b4b3aa8 100644 --- a/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-provided-result.yaml +++ b/comp/otelcol/ddflareextension/impl/testdata/simple-dd/config-provided-result.yaml @@ -88,7 +88,7 @@ exporters: peer_service_aggregation: false peer_tags: [] peer_tags_aggregation: false - span_name_as_resource_name: true + span_name_as_resource_name: false span_name_remappings: {} trace_buffer: 0 write_buffer_size: 0 diff --git a/comp/otelcol/logsagentpipeline/go.mod b/comp/otelcol/logsagentpipeline/go.mod index fd1925f5a1a1d..302cf0cf4787c 100644 --- a/comp/otelcol/logsagentpipeline/go.mod +++ b/comp/otelcol/logsagentpipeline/go.mod @@ -121,7 +121,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -153,7 +153,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/otelcol/logsagentpipeline/go.sum b/comp/otelcol/logsagentpipeline/go.sum index 9061e9b89a5c6..19075c11eaba4 100644 --- a/comp/otelcol/logsagentpipeline/go.sum +++ b/comp/otelcol/logsagentpipeline/go.sum @@ -99,8 +99,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod index 6bce0b3c962d4..9e6bc558c4423 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod @@ -136,7 +136,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -165,7 +165,7 @@ require ( go.uber.org/dig v1.18.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum index 9061e9b89a5c6..19075c11eaba4 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum @@ -99,8 +99,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/factory.go b/comp/otelcol/otlp/components/exporter/datadogexporter/factory.go index 35cb0957daada..b32933f255ba3 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/factory.go +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/factory.go @@ -119,7 +119,6 @@ func NewFactory( func CreateDefaultConfig() component.Config { ddcfg := datadogconfig.CreateDefaultConfig().(*datadogconfig.Config) ddcfg.Traces.TracesConfig.ComputeTopLevelBySpanKind = true - ddcfg.Traces.TracesConfig.SpanNameAsResourceName = true ddcfg.Logs.Endpoint = "https://agent-http-intake.logs.datadoghq.com" ddcfg.HostMetadata.Enabled = false return ddcfg diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index 5023166f00df5..b7d7886ae3aac 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../../../core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../../../../def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../../../../forwarder/defaultforwarder @@ -138,6 +139,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/status v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 // indirect @@ -239,7 +241,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect @@ -319,7 +321,7 @@ require ( go.uber.org/fx v1.23.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index 5107430987dd3..a86b1d8b49cf7 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -175,8 +175,8 @@ github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -534,8 +534,8 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod index 23ddd3225d779..1bd29cf7cf9ea 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod @@ -110,7 +110,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect @@ -157,7 +157,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum index 85a19bf8e57b5..d1856ceabaf7e 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum @@ -121,8 +121,8 @@ github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -377,8 +377,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod index 3ed3c5a02c175..1c14c95bf9c38 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod @@ -161,7 +161,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect @@ -215,7 +215,7 @@ require ( go.uber.org/fx v1.23.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum b/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum index b0e60957170e6..0b852b1fd8dd7 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum @@ -139,8 +139,8 @@ github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -460,8 +460,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/otlp/components/metricsclient/go.sum b/comp/otelcol/otlp/components/metricsclient/go.sum index 08cca2d402cd6..a5181f913d8e8 100644 --- a/comp/otelcol/otlp/components/metricsclient/go.sum +++ b/comp/otelcol/otlp/components/metricsclient/go.sum @@ -64,8 +64,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod index 86f54c29c25ad..0faa1b34bcd0e 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod @@ -79,9 +79,9 @@ require ( go.opentelemetry.io/otel/sdk v1.32.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.31.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum index 30f9f975f43b2..05605cfdfae08 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum @@ -107,20 +107,20 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 05eb43fdf4629..cff45d337dcd1 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -3,6 +3,7 @@ module github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsproces go 1.22.0 replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../metricsclient github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../../../../comp/trace/compression/impl-gzip @@ -33,6 +34,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.3 // indirect @@ -89,7 +91,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect @@ -97,7 +99,6 @@ require ( google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index 25a9e6fccac3c..abd898f9bed2a 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -196,8 +196,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -245,8 +245,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/comp/otelcol/otlp/testutil/go.mod b/comp/otelcol/otlp/testutil/go.mod index 58832aca7482e..609205dc9e161 100644 --- a/comp/otelcol/otlp/testutil/go.mod +++ b/comp/otelcol/otlp/testutil/go.mod @@ -72,7 +72,7 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect @@ -98,7 +98,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/otlp/testutil/go.sum b/comp/otelcol/otlp/testutil/go.sum index 22f79264c82e8..dca00de17ecd0 100644 --- a/comp/otelcol/otlp/testutil/go.sum +++ b/comp/otelcol/otlp/testutil/go.sum @@ -98,8 +98,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -293,8 +293,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/process/apiserver/apiserver.go b/comp/process/apiserver/apiserver.go index c216b2a93fef4..9c37154296f64 100644 --- a/comp/process/apiserver/apiserver.go +++ b/comp/process/apiserver/apiserver.go @@ -15,8 +15,11 @@ import ( "go.uber.org/fx" "github.com/DataDog/datadog-agent/cmd/process-agent/api" - log "github.com/DataDog/datadog-agent/comp/core/log/def" + "github.com/DataDog/datadog-agent/comp/api/authtoken" + logComp "github.com/DataDog/datadog-agent/comp/core/log/def" + "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/log" ) var _ Component = (*apiserver)(nil) @@ -30,7 +33,9 @@ type dependencies struct { Lc fx.Lifecycle - Log log.Component + Log logComp.Component + + At authtoken.Component APIServerDeps api.APIServerDeps } @@ -38,6 +43,7 @@ type dependencies struct { //nolint:revive // TODO(PROC) Fix revive linter func newApiServer(deps dependencies) Component { r := mux.NewRouter() + r.Use(validateToken) api.SetupAPIServerHandlers(deps.APIServerDeps, r) // Set up routes addr, err := pkgconfigsetup.GetProcessAPIAddressPort(pkgconfigsetup.Datadog()) @@ -54,6 +60,7 @@ func newApiServer(deps dependencies) Component { ReadTimeout: timeout, WriteTimeout: timeout, IdleTimeout: timeout, + TLSConfig: deps.At.GetTLSServerConfig(), }, } @@ -80,3 +87,13 @@ func newApiServer(deps dependencies) Component { return apiserver } + +func validateToken(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := util.Validate(w, r); err != nil { + log.Warnf("invalid auth token for %s request to %s: %s", r.Method, r.RequestURI, err) + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/comp/process/apiserver/apiserver_test.go b/comp/process/apiserver/apiserver_test.go index c9b0f4b96e92d..1131bad880776 100644 --- a/comp/process/apiserver/apiserver_test.go +++ b/comp/process/apiserver/apiserver_test.go @@ -6,14 +6,19 @@ package apiserver import ( + "fmt" + "net" "net/http" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" "github.com/DataDog/datadog-agent/comp/core" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/settings/settingsimpl" "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/core/status/statusimpl" @@ -21,13 +26,21 @@ import ( taggerfx "github.com/DataDog/datadog-agent/comp/core/tagger/fx" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx" + "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) func TestLifecycle(t *testing.T) { + listener, err := net.Listen("tcp", ":0") + require.NoError(t, err) + port := listener.Addr().(*net.TCPAddr).Port + _ = fxutil.Test[Component](t, fx.Options( Module(), core.MockBundle(), + fx.Replace(config.MockParams{Overrides: map[string]interface{}{ + "process_config.cmd_port": port, + }}), workloadmetafx.Module(workloadmeta.NewParams()), fx.Supply( status.Params{ @@ -39,15 +52,61 @@ func TestLifecycle(t *testing.T) { }), statusimpl.Module(), settingsimpl.MockModule(), + createandfetchimpl.Module(), )) - assert.Eventually(t, func() bool { - res, err := http.Get("http://localhost:6162/config") - if err != nil { - return false - } + assert.EventuallyWithT(t, func(c *assert.CollectT) { + url := fmt.Sprintf("http://localhost:%d/agent/status", port) + req, err := http.NewRequest("GET", url, nil) + require.NoError(c, err) + req.Header.Set("Authorization", "Bearer "+util.GetAuthToken()) + res, err := util.GetClient(false).Do(req) + require.NoError(c, err) defer res.Body.Close() + assert.Equal(c, http.StatusOK, res.StatusCode) + }, 5*time.Second, time.Second) +} - return res.StatusCode == http.StatusOK +func TestPostAuthentication(t *testing.T) { + listener, err := net.Listen("tcp", ":0") + require.NoError(t, err) + port := listener.Addr().(*net.TCPAddr).Port + + _ = fxutil.Test[Component](t, fx.Options( + Module(), + core.MockBundle(), + fx.Replace(config.MockParams{Overrides: map[string]interface{}{ + "process_config.cmd_port": port, + }}), + workloadmetafx.Module(workloadmeta.NewParams()), + fx.Supply( + status.Params{ + PythonVersionGetFunc: func() string { return "n/a" }, + }, + ), + taggerfx.Module(tagger.Params{ + UseFakeTagger: true, + }), + statusimpl.Module(), + settingsimpl.MockModule(), + createandfetchimpl.Module(), + )) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + // No authentication + url := fmt.Sprintf("http://localhost:%d/config/log_level?value=debug", port) + req, err := http.NewRequest("POST", url, nil) + require.NoError(c, err) + res, err := util.GetClient(false).Do(req) + require.NoError(c, err) + defer res.Body.Close() + assert.Equal(c, http.StatusUnauthorized, res.StatusCode) + + // With authentication + req.Header.Set("Authorization", "Bearer "+util.GetAuthToken()) + res, err = util.GetClient(false).Do(req) + require.NoError(c, err) + defer res.Body.Close() + assert.Equal(c, http.StatusOK, res.StatusCode) }, 5*time.Second, time.Second) } diff --git a/comp/process/bundle_test.go b/comp/process/bundle_test.go index 591dce9f63cfb..9fb4131e013c0 100644 --- a/comp/process/bundle_test.go +++ b/comp/process/bundle_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" configComp "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" @@ -60,6 +61,7 @@ func TestBundleDependencies(t *testing.T) { rdnsquerier.MockModule(), npcollectorimpl.MockModule(), statsd.MockModule(), + fetchonlyimpl.MockModule(), ) } diff --git a/comp/remote-config/rcclient/rcclientimpl/rcclient.go b/comp/remote-config/rcclient/rcclientimpl/rcclient.go index 4e43c1d99b1aa..374bb41bfe459 100644 --- a/comp/remote-config/rcclient/rcclientimpl/rcclient.go +++ b/comp/remote-config/rcclient/rcclientimpl/rcclient.go @@ -168,6 +168,10 @@ func (rc rcClient) start() { } } +// mrfUpdateCallback is the callback function for the AGENT_FAILOVER configs. +// It fetches all the configs targeting the agent and applies the failover settings +// using an OR strategy. In case of nil the value is not updated, for a false it does not update if +// the setting is already set to true. func (rc rcClient) mrfUpdateCallback(updates map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) { // If the updates map is empty, we should unset the failover settings if they were set via RC previously if len(updates) == 0 { @@ -188,8 +192,13 @@ func (rc rcClient) mrfUpdateCallback(updates map[string]state.RawConfig, applySt return } - applied := false + var enableLogs, enableMetrics *bool + var enableLogsCfgPth, enableMetricsCfgPth string for cfgPath, update := range updates { + if (enableLogs != nil && *enableLogs) && (enableMetrics != nil && *enableMetrics) { + break + } + mrfUpdate, err := parseMultiRegionFailoverConfig(update.Config) if err != nil { pkglog.Errorf("Multi-Region Failover update unmarshal failed: %s", err) @@ -200,42 +209,55 @@ func (rc rcClient) mrfUpdateCallback(updates map[string]state.RawConfig, applySt continue } - if mrfUpdate != nil && (mrfUpdate.FailoverMetrics != nil || mrfUpdate.FailoverLogs != nil) { - // If we've received multiple config files updating the failover settings, we should disregard all but the first update and log it, as this is unexpected - if applied { - pkglog.Warnf("Multiple Multi-Region Failover updates received, disregarding update of `multi_region_failover.failover_metrics` to %v and `multi_region_failover.failover_logs` to %v", mrfUpdate.FailoverMetrics, mrfUpdate.FailoverLogs) - applyStateCallback(cfgPath, state.ApplyStatus{ - State: state.ApplyStateError, - Error: "Multiple Multi-Region Failover updates received. Only the first was applied.", - }) - continue - } + if mrfUpdate == nil || (mrfUpdate.FailoverMetrics == nil && mrfUpdate.FailoverLogs == nil) { + continue + } - if mrfUpdate.FailoverMetrics != nil { - err = rc.applyMRFRuntimeSetting("multi_region_failover.failover_metrics", *mrfUpdate.FailoverMetrics, cfgPath, applyStateCallback) - if err != nil { - continue - } - change := "disabled" - if *mrfUpdate.FailoverMetrics { - change = "enabled" - } - pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for metrics", change) - } - if mrfUpdate.FailoverLogs != nil { - err = rc.applyMRFRuntimeSetting("multi_region_failover.failover_logs", *mrfUpdate.FailoverLogs, cfgPath, applyStateCallback) - if err != nil { - continue - } - change := "disabled" - if *mrfUpdate.FailoverLogs { - change = "enabled" - } - pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for logs", change) - } - applyStateCallback(cfgPath, state.ApplyStatus{State: state.ApplyStateAcknowledged}) - applied = true + if !(enableMetrics != nil && *enableMetrics) && mrfUpdate.FailoverMetrics != nil { + enableMetrics = mrfUpdate.FailoverMetrics + enableMetricsCfgPth = cfgPath + } + + if !(enableLogs != nil && *enableLogs) && mrfUpdate.FailoverLogs != nil { + enableLogs = mrfUpdate.FailoverLogs + enableLogsCfgPth = cfgPath + } + } + + if enableMetrics != nil { + err := rc.applyMRFRuntimeSetting("multi_region_failover.failover_metrics", *enableMetrics, enableMetricsCfgPth, applyStateCallback) + if err != nil { + pkglog.Errorf("Multi-Region Failover failed to apply new metrics settings : %s", err) + applyStateCallback(enableMetricsCfgPth, state.ApplyStatus{ + State: state.ApplyStateError, + Error: err.Error(), + }) + return + } + change := "disabled" + if *enableMetrics { + change = "enabled" + } + pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for metrics", change) + applyStateCallback(enableMetricsCfgPth, state.ApplyStatus{State: state.ApplyStateAcknowledged}) + } + + if enableLogs != nil { + err := rc.applyMRFRuntimeSetting("multi_region_failover.failover_logs", *enableLogs, enableLogsCfgPth, applyStateCallback) + if err != nil { + pkglog.Errorf("Multi-Region Failover failed to apply new logs settings : %s", err) + applyStateCallback(enableMetricsCfgPth, state.ApplyStatus{ + State: state.ApplyStateError, + Error: err.Error(), + }) + return + } + change := "disabled" + if *enableLogs { + change = "enabled" } + pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for logs", change) + applyStateCallback(enableLogsCfgPth, state.ApplyStatus{State: state.ApplyStateAcknowledged}) } } diff --git a/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go b/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go index f6974273d3325..71b8bca9de2f2 100644 --- a/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go +++ b/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go @@ -6,6 +6,7 @@ package rcclientimpl import ( + "fmt" "testing" "time" @@ -61,6 +62,39 @@ func (m *mockLogLevelRuntimeSettings) Hidden() bool { func applyEmpty(_ string, _ state.ApplyStatus) {} +type MockComponent interface { + settings.Component + + SetRuntimeSetting(setting string, value interface{}, source model.Source) error +} + +type MockComponentImplMrf struct { + settings.Component + + logs *bool + metrics *bool + traces *bool +} + +func (m *MockComponentImplMrf) SetRuntimeSetting(setting string, value interface{}, _ model.Source) error { + v, ok := value.(bool) + if !ok { + return fmt.Errorf("unexpected value type %T", value) + } + + switch setting { + case "multi_region_failover.failover_metrics": + m.metrics = &v + case "multi_region_failover.failover_logs": + m.logs = &v + case "multi_region_failover.failover_traces": + m.traces = &v + default: + return &settings.SettingNotFoundError{Name: setting} + } + return nil +} + func TestRCClientCreate(t *testing.T) { _, err := newRemoteConfigClient( fxutil.Test[dependencies]( @@ -189,3 +223,61 @@ func TestAgentConfigCallback(t *testing.T) { assert.Equal(t, "debug", cfg.Get("log_level")) assert.Equal(t, model.SourceCLI, cfg.GetSource("log_level")) } + +func TestAgentMRFConfigCallback(t *testing.T) { + pkglog.SetupLogger(pkglog.Default(), "info") + cfg := configmock.New(t) + + rc := fxutil.Test[rcclient.Component](t, + fx.Options( + Module(), + fx.Provide(func() log.Component { return logmock.New(t) }), + fx.Provide(func() config.Component { return cfg }), + sysprobeconfig.NoneModule(), + fx.Supply( + rcclient.Params{ + AgentName: "test-agent", + AgentVersion: "7.0.0", + }, + ), + fx.Supply( + settings.Params{ + Settings: map[string]settings.RuntimeSetting{ + "log_level": &mockLogLevelRuntimeSettings{logLevel: "info"}, + }, + Config: cfg, + }, + ), + settingsimpl.Module(), + ), + ) + + allInactive := state.RawConfig{Config: []byte(`{"name": "none"}`)} + noLogs := state.RawConfig{Config: []byte(`{"name": "nologs", "failover_logs": false}`)} + activeMetrics := state.RawConfig{Config: []byte(`{"name": "yesmetrics", "failover_metrics": true}`)} + + structRC := rc.(rcClient) + + ipcAddress, err := pkgconfigsetup.GetIPCAddress(cfg) + assert.NoError(t, err) + + structRC.client, _ = client.NewUnverifiedGRPCClient( + ipcAddress, pkgconfigsetup.GetIPCPort(), func() (string, error) { return security.FetchAuthToken(cfg) }, + client.WithAgent("test-agent", "9.99.9"), + client.WithProducts(state.ProductAgentConfig), + client.WithPollInterval(time.Hour), + ) + structRC.settingsComponent = &MockComponentImplMrf{} + + // Should enable metrics failover and disable logs failover + structRC.mrfUpdateCallback(map[string]state.RawConfig{ + "datadog/2/AGENT_FAILOVER/none/configname": allInactive, + "datadog/2/AGENT_FAILOVER/nologs/configname": noLogs, + "datadog/2/AGENT_FAILOVER/yesmetrics/configname": activeMetrics, + }, applyEmpty) + + cmpntSettings := structRC.settingsComponent.(*MockComponentImplMrf) + assert.True(t, *cmpntSettings.metrics) + assert.False(t, *cmpntSettings.logs) + assert.Nil(t, cmpntSettings.traces) +} diff --git a/comp/serializer/compression/go.mod b/comp/serializer/compression/go.mod index a46ee59b26ae6..f1c59cd40d645 100644 --- a/comp/serializer/compression/go.mod +++ b/comp/serializer/compression/go.mod @@ -71,7 +71,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/comp/serializer/compression/go.sum b/comp/serializer/compression/go.sum index a3f8c86e6fc2e..3b405d22ebcc8 100644 --- a/comp/serializer/compression/go.sum +++ b/comp/serializer/compression/go.sum @@ -89,8 +89,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/comp/snmptraps/listener/listenerimpl/listener.go b/comp/snmptraps/listener/listenerimpl/listener.go index 7571896edc30d..ab3d5c3aa3dea 100644 --- a/comp/snmptraps/listener/listenerimpl/listener.go +++ b/comp/snmptraps/listener/listenerimpl/listener.go @@ -8,6 +8,7 @@ package listenerimpl import ( "context" + "crypto/subtle" "errors" "fmt" "net" @@ -167,7 +168,8 @@ func validatePacket(p *gosnmp.SnmpPacket, c *config.TrapsConfig) error { // At least one of the known community strings must match. for _, community := range c.CommunityStrings { - if community == p.Community { + // Simple string equality check, but in constant time to avoid timing attacks + if subtle.ConstantTimeCompare([]byte(community), []byte(p.Community)) == 1 { return nil } } diff --git a/comp/trace/agent/def/go.mod b/comp/trace/agent/def/go.mod index 56d13f69c88cf..1bfc5a3ba75af 100644 --- a/comp/trace/agent/def/go.mod +++ b/comp/trace/agent/def/go.mod @@ -26,7 +26,7 @@ require ( go.opentelemetry.io/otel/trace v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/trace/agent/def/go.sum b/comp/trace/agent/def/go.sum index 7b3dbbc7fe0b3..92539a14a3bd1 100644 --- a/comp/trace/agent/def/go.sum +++ b/comp/trace/agent/def/go.sum @@ -82,8 +82,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/trace/agent/impl/agent.go b/comp/trace/agent/impl/agent.go index b3d783288ceb6..de9237dedeea5 100644 --- a/comp/trace/agent/impl/agent.go +++ b/comp/trace/agent/impl/agent.go @@ -24,6 +24,7 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" @@ -68,6 +69,7 @@ type dependencies struct { Statsd statsd.Component Tagger tagger.Component Compressor compression.Component + At authtoken.Component } var _ traceagent.Component = (*component)(nil) @@ -93,6 +95,7 @@ type component struct { params *Params tagger tagger.Component telemetryCollector telemetry.TelemetryCollector + at authtoken.Component wg *sync.WaitGroup } @@ -115,6 +118,7 @@ func NewAgent(deps dependencies) (traceagent.Component, error) { params: deps.Params, telemetryCollector: deps.TelemetryCollector, tagger: deps.Tagger, + at: deps.At, wg: &sync.WaitGroup{}, } statsdCl, err := setupMetrics(deps.Statsd, c.config, c.telemetryCollector) diff --git a/comp/trace/agent/impl/run.go b/comp/trace/agent/impl/run.go index 20ecbc6496a1b..f40b36e29ae7f 100644 --- a/comp/trace/agent/impl/run.go +++ b/comp/trace/agent/impl/run.go @@ -98,6 +98,9 @@ func runAgentSidekicks(ag component) error { })) } + // Configure the Trace Agent Debug server to use the IPC certificate + ag.Agent.DebugServer.SetTLSConfig(ag.at.GetTLSServerConfig()) + log.Infof("Trace agent running on host %s", tracecfg.Hostname) if pcfg := profilingConfig(tracecfg); pcfg != nil { if err := profiling.Start(*pcfg); err != nil { diff --git a/comp/trace/bundle_test.go b/comp/trace/bundle_test.go index e9874a4b40077..692e7c255f473 100644 --- a/comp/trace/bundle_test.go +++ b/comp/trace/bundle_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" coreconfig "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" @@ -45,6 +47,7 @@ func TestBundleDependencies(t *testing.T) { zstdfx.Module(), taggerfx.Module(tagger.Params{}), fx.Supply(&traceagentimpl.Params{}), + createandfetchimpl.Module(), ) } @@ -75,6 +78,7 @@ func TestMockBundleDependencies(t *testing.T) { fx.Invoke(func(_ traceagent.Component) {}), MockBundle(), taggerfx.Module(tagger.Params{}), + fetchonlyimpl.MockModule(), )) require.NotNil(t, cfg.Object()) diff --git a/comp/trace/config/config_test.go b/comp/trace/config/config_test.go index 9c9d45eb284ac..9706e2c784f4c 100644 --- a/comp/trace/config/config_test.go +++ b/comp/trace/config/config_test.go @@ -9,6 +9,7 @@ import ( "bufio" "bytes" "context" + _ "embed" "encoding/json" "errors" "net/http" @@ -245,6 +246,9 @@ func TestTelemetryEndpointsConfig(t *testing.T) { }) } +//go:embed testdata/stringcode.go.tmpl +var stringCodeBody string + func TestConfigHostname(t *testing.T) { t.Run("fail", func(t *testing.T) { overrides := map[string]interface{}{ @@ -360,11 +364,6 @@ func TestConfigHostname(t *testing.T) { }) t.Run("external", func(t *testing.T) { - body, err := os.ReadFile("testdata/stringcode.go.tmpl") - if err != nil { - t.Fatal(err) - } - // makeProgram creates a new binary file which returns the given response and exits to the OS // given the specified code, returning the path of the program. makeProgram := func(t *testing.T, response string, code int) string { @@ -372,7 +371,7 @@ func TestConfigHostname(t *testing.T) { if err != nil { t.Fatal(err) } - tmpl, err := template.New("program").Parse(string(body)) + tmpl, err := template.New("program").Parse(stringCodeBody) if err != nil { t.Fatal(err) } @@ -399,6 +398,7 @@ func TestConfigHostname(t *testing.T) { fallbackHostnameFunc = func() (string, error) { return "fallback.host", nil } t.Run("good", func(t *testing.T) { + t.Skip("Skip flaky test while we explore fixes.") bin := makeProgram(t, "host.name", 0) defer os.Remove(bin) diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index f19c5eeaf8a4f..06a157dc6418b 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -19,13 +19,14 @@ import ( "go.opentelemetry.io/collector/component/componenttest" - apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" corecompcfg "github.com/DataDog/datadog-agent/comp/core/config" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/configcheck" + apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/env" "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -121,6 +122,9 @@ func prepareConfig(c corecompcfg.Component, tagger tagger.Component) (*config.Ag cfg.ContainerTags = func(cid string) ([]string, error) { return tagger.Tag(types.NewEntityID(types.ContainerID, cid), types.HighCardinality) } + cfg.ContainerIDFromOriginInfo = func(originInfo origindetection.OriginInfo) (string, error) { + return tagger.GenerateContainerIDFromOriginInfo(originInfo) + } cfg.ContainerProcRoot = coreConfigObject.GetString("container_proc_root") cfg.GetAgentAuthToken = apiutil.GetAuthToken return cfg, nil diff --git a/comp/trace/status/statusimpl/status.go b/comp/trace/status/statusimpl/status.go index e476ee0281d7a..00a8730b87da8 100644 --- a/comp/trace/status/statusimpl/status.go +++ b/comp/trace/status/statusimpl/status.go @@ -95,7 +95,7 @@ func (s statusProvider) populateStatus() map[string]interface{} { port := s.Config.GetInt("apm_config.debug.port") c := client() - url := fmt.Sprintf("http://localhost:%d/debug/vars", port) + url := fmt.Sprintf("https://localhost:%d/debug/vars", port) resp, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return map[string]interface{}{ diff --git a/comp/updater/telemetry/telemetryimpl/telemetry.go b/comp/updater/telemetry/telemetryimpl/telemetry.go index d8f961e8398e6..ee1bab4efdd56 100644 --- a/comp/updater/telemetry/telemetryimpl/telemetry.go +++ b/comp/updater/telemetry/telemetryimpl/telemetry.go @@ -7,10 +7,10 @@ package telemetryimpl import ( + "context" "net/http" "go.uber.org/fx" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/updater/telemetry" @@ -38,13 +38,7 @@ func newTelemetry(deps dependencies) (telemetry.Component, error) { client := &http.Client{ Transport: httputils.CreateHTTPTransport(deps.Config), } - telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", - fleettelemetry.WithSamplingRules( - tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1), - tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05), - tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05), - ), - ) - deps.Lc.Append(fx.Hook{OnStart: telemetry.Start, OnStop: telemetry.Stop}) + telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon") + deps.Lc.Append(fx.Hook{OnStop: func(context.Context) error { telemetry.Stop(); return nil }}) return telemetry, nil } diff --git a/docs/cloud-workload-security/backend_linux.md b/docs/cloud-workload-security/backend_linux.md index 7b0794f82fc9d..e4e1af8c22c37 100644 --- a/docs/cloud-workload-security/backend_linux.md +++ b/docs/cloud-workload-security/backend_linux.md @@ -1718,9 +1718,6 @@ CSM Threats event for Linux systems have the following JSON schema: "container": { "$ref": "#/$defs/ContainerContext" }, - "cgroup": { - "$ref": "#/$defs/CGroupContext" - }, "network": { "$ref": "#/$defs/NetworkContext" }, @@ -1730,6 +1727,9 @@ CSM Threats event for Linux systems have the following JSON schema: "security_profile": { "$ref": "#/$defs/SecurityProfileContext" }, + "cgroup": { + "$ref": "#/$defs/CGroupContext" + }, "selinux": { "$ref": "#/$defs/SELinuxEvent" }, @@ -1802,10 +1802,10 @@ CSM Threats event for Linux systems have the following JSON schema: | `exit` | $ref | Please see [ExitEvent](#exitevent) | | `process` | $ref | Please see [ProcessContext](#processcontext) | | `container` | $ref | Please see [ContainerContext](#containercontext) | -| `cgroup` | $ref | Please see [CGroupContext](#cgroupcontext) | | `network` | $ref | Please see [NetworkContext](#networkcontext) | | `dd` | $ref | Please see [DDContext](#ddcontext) | | `security_profile` | $ref | Please see [SecurityProfileContext](#securityprofilecontext) | +| `cgroup` | $ref | Please see [CGroupContext](#cgroupcontext) | | `selinux` | $ref | Please see [SELinuxEvent](#selinuxevent) | | `bpf` | $ref | Please see [BPFEvent](#bpfevent) | | `mmap` | $ref | Please see [MMapEvent](#mmapevent) | diff --git a/docs/cloud-workload-security/backend_linux.schema.json b/docs/cloud-workload-security/backend_linux.schema.json index b6f9cddc6cbb7..b4aa6b761fbfb 100644 --- a/docs/cloud-workload-security/backend_linux.schema.json +++ b/docs/cloud-workload-security/backend_linux.schema.json @@ -1707,9 +1707,6 @@ "container": { "$ref": "#/$defs/ContainerContext" }, - "cgroup": { - "$ref": "#/$defs/CGroupContext" - }, "network": { "$ref": "#/$defs/NetworkContext" }, @@ -1719,6 +1716,9 @@ "security_profile": { "$ref": "#/$defs/SecurityProfileContext" }, + "cgroup": { + "$ref": "#/$defs/CGroupContext" + }, "selinux": { "$ref": "#/$defs/SELinuxEvent" }, diff --git a/flakes.yaml b/flakes.yaml index bde9554d2f5f7..8c95e78ab01e0 100644 --- a/flakes.yaml +++ b/flakes.yaml @@ -11,8 +11,4 @@ test/new-e2e/tests/containers: - TestECSSuite/TestCPU/metric___container.cpu.usage{^ecs_container_name:stress-ng$} - TestEKSSuite/TestCPU/metric___container.cpu.usage{^kube_deployment:stress-ng$,^kube_namespace:workload-cpustress$} - TestKindSuite/TestCPU/metric___container.cpu.usage{^kube_deployment:stress-ng$,^kube_namespace:workload-cpustress$} - - - TestEKSSuite/TestDogstatsdInAgent/metric___custom.metric{^kube_deployment:dogstatsd-udp$,^kube_namespace:workload-dogstatsd$} - - TestEKSSuite/TestDogstatsdStandalone/metric___custom.metric{^kube_deployment:dogstatsd-udp$,^kube_namespace:workload-dogstatsd$} - - TestKindSuite/TestDogstatsdInAgent/metric___custom.metric{^kube_deployment:dogstatsd-udp$,^kube_namespace:workload-dogstatsd$} - - TestKindSuite/TestDogstatsdStandalone/metric___custom.metric{^kube_deployment:dogstatsd-udp$,^kube_namespace:workload-dogstatsd$} + - TestKindSuite/TestAdmissionControllerWithAutoDetectedLanguage diff --git a/go.mod b/go.mod index eac05ebaab918..b691c2ef988b2 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/secrets => ./comp/core/secrets github.com/DataDog/datadog-agent/comp/core/status => ./comp/core/status github.com/DataDog/datadog-agent/comp/core/status/statusimpl => ./comp/core/status/statusimpl + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ./comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/tagger/tags => ./comp/core/tagger/tags github.com/DataDog/datadog-agent/comp/core/tagger/types => ./comp/core/tagger/types github.com/DataDog/datadog-agent/comp/core/tagger/utils => ./comp/core/tagger/utils @@ -226,7 +227,7 @@ require ( github.com/hashicorp/consul/api v1.30.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb github.com/iceber/iouring-go v0.0.0-20230403020409-002cfd2e2a90 github.com/imdario/mergo v0.3.16 github.com/invopop/jsonschema v0.12.0 @@ -261,7 +262,6 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/samber/lo v1.47.0 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/shirou/gopsutil/v3 v3.24.5 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 github.com/sirupsen/logrus v1.9.3 github.com/skydive-project/go-debouncer v1.0.0 @@ -307,7 +307,7 @@ require ( go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d golang.org/x/arch v0.12.0 golang.org/x/exp v0.0.0-20241210194714-1829a127f884 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.28.0 golang.org/x/text v0.21.0 @@ -374,19 +374,19 @@ require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.32.5 - github.com/aws/aws-sdk-go-v2/config v1.28.5 - github.com/aws/aws-sdk-go-v2/credentials v1.17.46 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/service/ebs v1.27.0 // indirect github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect @@ -425,7 +425,6 @@ require ( github.com/godbus/dbus/v5 v5.1.0 github.com/golang/glog v1.2.2 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/licenseclassifier/v2 v2.0.0 // indirect github.com/google/uuid v1.6.0 github.com/google/wire v0.6.0 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect @@ -655,6 +654,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 github.com/DataDog/datadog-agent/comp/core/status v0.59.0-rc.6 github.com/DataDog/datadog-agent/comp/core/status/statusimpl v0.56.0-rc.3 + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/DataDog/datadog-agent/comp/core/tagger/tags v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/core/tagger/types v0.59.0 github.com/DataDog/datadog-agent/comp/core/telemetry v0.59.0 @@ -975,7 +975,6 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/signalfx/sapm-proto v0.17.0 // indirect github.com/sigstore/rekor v1.2.2 // indirect github.com/skeema/knownhosts v1.2.2 // indirect @@ -1065,7 +1064,7 @@ replace github.com/vishvananda/netlink => github.com/DataDog/netlink v1.0.1-0.20 // Pull in replacements needed by upstream Trivy replace ( // Maps to Trivy fork https://github.com/DataDog/trivy/commits/use-fs-main-dd/ - github.com/aquasecurity/trivy => github.com/DataDog/trivy v0.0.0-20241126101205-8517f9b946f4 + github.com/aquasecurity/trivy => github.com/DataDog/trivy v0.0.0-20241216135157-95e0e96002ee github.com/saracen/walker => github.com/DataDog/walker v0.0.0-20230418153152-7f29bb2dc950 // testcontainers-go has a bug with versions v0.25.0 and v0.26.0 // ref: https://github.com/testcontainers/testcontainers-go/issues/1782 diff --git a/go.sum b/go.sum index 9bc58e297e1b0..a3c3cdbec81a7 100644 --- a/go.sum +++ b/go.sum @@ -174,8 +174,8 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.22.0 h1:63SzQz9Ab8XJ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.22.0/go.mod h1:E/PY/aQ6S/N5hBPHXZRGmovs5b1BSi4RHGNcB4yP/Z0= github.com/DataDog/sketches-go v1.4.6 h1:acd5fb+QdUzGrosfNLwrIhqyrbMORpvBy7mE+vHlT3I= github.com/DataDog/sketches-go v1.4.6/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= -github.com/DataDog/trivy v0.0.0-20241126101205-8517f9b946f4 h1:UVL5oU/8o0JhEv8Js6qxJgiqeV+PzPw/aldojyexS/U= -github.com/DataDog/trivy v0.0.0-20241126101205-8517f9b946f4/go.mod h1:hLiUAm3v175M5jWbq34TdGmX6mvHIJY7FMuZ3wBugtw= +github.com/DataDog/trivy v0.0.0-20241216135157-95e0e96002ee h1:taj22FDHhWs9QkVJypOYMhdgJTnTb5y4SHiW7YcJJms= +github.com/DataDog/trivy v0.0.0-20241216135157-95e0e96002ee/go.mod h1:hv4bqUzcHXSfVft8E9DUlu9d+szHrxYSh+WykZU70Dk= github.com/DataDog/viper v1.14.0 h1:dIjTe/uJiah+QFqFZ+MXeqgmUvWhg37l37ZxFWxr3is= github.com/DataDog/viper v1.14.0/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/walker v0.0.0-20230418153152-7f29bb2dc950 h1:2imDajw3V85w1iqHsuXN+hUBZQVF+r9eME8tsPq/HpA= @@ -321,24 +321,24 @@ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= -github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= -github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY= github.com/aws/aws-sdk-go-v2/config v1.19.0/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= -github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= -github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= @@ -352,8 +352,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhv github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA= github.com/aws/aws-sdk-go-v2/service/kms v1.37.6 h1:CZImQdb1QbU9sGgJ9IswhVkxAcjkkD1eQTMA1KHWk+E= github.com/aws/aws-sdk-go-v2/service/kms v1.37.6/go.mod h1:YJDdlK0zsyxVBxGU48AR/Mi8DMrGdc1E3Yij4fNrONA= @@ -363,14 +363,14 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2/go.mod h1:Zjfqt7KhQK+PO1bbOsFNzK github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.6 h1:1KDMKvOKNrpD667ORbZ/+4OgvUoaok1gg/MLzrHF9fw= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.6/go.mod h1:DmtyfCfONhOyVAJ6ZMTrDSFIeyCBlEO93Qkfhxwbxu0= github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= @@ -843,8 +843,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA= -github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -972,8 +970,8 @@ github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 h1:fgVfQ4AC1av github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/hetznercloud/hcloud-go/v2 v2.10.2 h1:9gyTUPhfNbfbS40Spgij5mV5k37bOZgt8iHKCbfGs5I= github.com/hetznercloud/hcloud-go/v2 v2.10.2/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -1607,7 +1605,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA= github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= @@ -2261,8 +2258,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/go.work b/go.work index 880b236237204..c7c6ebf27d3ff 100644 --- a/go.work +++ b/go.work @@ -17,6 +17,7 @@ use ( comp/core/secrets comp/core/status comp/core/status/statusimpl + comp/core/tagger/origindetection comp/core/tagger/tags comp/core/tagger/types comp/core/tagger/utils diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 79be9ca61a043..3b03d5fa6b2af 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -227,7 +227,7 @@ require ( golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/internal/tools/go.sum b/internal/tools/go.sum index 2bf7f8b759bd6..e3f2dd4d19710 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -643,8 +643,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20170207211851-4464e7848382/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/perf v0.0.0-20210220033136-40a54f11e909 h1:rWw0Gj4DMl/2otJ8CnfTcwOWkpROAc6qhXXoMrYOCgo= golang.org/x/perf v0.0.0-20210220033136-40a54f11e909/go.mod h1:KRSrLY7jerMEa0Ih7gBheQ3FYDiSx6liMnniX1o3j2g= diff --git a/internal/tools/proto/go.mod b/internal/tools/proto/go.mod index 12d0a2fb58004..4c81cd0d2261a 100644 --- a/internal/tools/proto/go.mod +++ b/internal/tools/proto/go.mod @@ -19,7 +19,7 @@ require ( github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect diff --git a/internal/tools/proto/go.sum b/internal/tools/proto/go.sum index 3d105f38523d1..43d99cf5a50b1 100644 --- a/internal/tools/proto/go.sum +++ b/internal/tools/proto/go.sum @@ -71,8 +71,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/modules.yml b/modules.yml index f9550b4b82943..5d5e5cb2057ff 100644 --- a/modules.yml +++ b/modules.yml @@ -35,6 +35,8 @@ modules: comp/core/status: used_by_otel: true comp/core/status/statusimpl: default + comp/core/tagger/origindetection: + used_by_otel: true comp/core/tagger/tags: used_by_otel: true comp/core/tagger/types: diff --git a/omnibus/config/software/openscap.rb b/omnibus/config/software/openscap.rb index e97250467e2a2..3e8e08d71990d 100644 --- a/omnibus/config/software/openscap.rb +++ b/omnibus/config/software/openscap.rb @@ -4,12 +4,12 @@ # Copyright 2016-present Datadog, Inc. name 'openscap' -default_version '1.3.10' +default_version '1.4.0' license "LGPL-3.0-or-later" license_file "COPYING" -version("1.3.10") { source sha256: "0d023ff3fbdec617768ea5977fd3bb6702dfef4ae595da9a5bbc6ecc6ac9e575" } +version("1.4.0") { source sha256: "b64f3709f0e072262708212d56897c419abdd2dc618273a363bc0497c6b4ba0b" } ship_source_offer true diff --git a/omnibus/config/software/openssl-fips-provider.rb b/omnibus/config/software/openssl-fips-provider.rb index 776d7e81531b3..7d9584b2e3767 100644 --- a/omnibus/config/software/openssl-fips-provider.rb +++ b/omnibus/config/software/openssl-fips-provider.rb @@ -22,14 +22,16 @@ relative_path "openssl-#{OPENSSL_FIPS_MODULE_VERSION}" build do + env = with_standard_compiler_flags() + env["MAKEFLAGS"] = "-j#{workers}" # Exact build steps from security policy: # https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp4282.pdf # # ---------------- DO NOT MODIFY LINES BELOW HERE ---------------- - command "./Configure enable-fips" + command "./Configure enable-fips", env: env - command "make" - command "make install" + command "make", env: env + command "make install", env: env # ---------------- DO NOT MODIFY LINES ABOVE HERE ---------------- mkdir "#{install_dir}/embedded/ssl" diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 7f8fd8e8f4219..1cd3c096363f2 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -54,10 +54,12 @@ # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default # compiler we use would end up being available in the system/docker image used by customers - if linux_target? + if linux_target? && env["CC"] command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env - command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env + end + if linux_target? && env["CXX"] + command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end delete "#{install_dir}/embedded/lib/python#{major}.#{minor}/test" @@ -79,7 +81,6 @@ license "Python-2.0" command "XCOPY /YEHIR *.* \"#{windows_safe_path(python_3_embedded)}\"" - command "copy /y \"#{windows_safe_path(vcrt140_root)}\\*.dll\" \"#{windows_safe_path(python_3_embedded)}\"" # Install pip python = "#{windows_safe_path(python_3_embedded)}\\python.exe" diff --git a/omnibus/config/software/vc_redist_14.rb b/omnibus/config/software/vc_redist_14.rb index ecc8067943f96..226a1145bb993 100644 --- a/omnibus/config/software/vc_redist_14.rb +++ b/omnibus/config/software/vc_redist_14.rb @@ -23,5 +23,5 @@ else source_msm = "Microsoft_VC141_CRT_x64.msm" end - command "powershell -C \"#{windows_safe_path(script_root)} -file #{source_msm} -targetDir .\\expanded\"" + command "powershell -C \"#{windows_safe_path(script_root)} -file #{source_msm} -targetDir \"#{windows_safe_path(python_3_embedded)}\"\"" end diff --git a/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb b/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb index 1304d1021b08d..23b1637b9c017 100644 --- a/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb +++ b/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb @@ -5,6 +5,7 @@ openssl_conf = openssl_init [openssl_init] providers = provider_sect +alg_section = algorithm_sect [provider_sect] fips = fips_sect @@ -12,3 +13,6 @@ base = base_sect [base_sect] activate = 1 + +[algorithm_sect] +default_properties = fips=yes diff --git a/pkg/api/go.mod b/pkg/api/go.mod index c3499a2fb94c2..ab50796cf7a58 100644 --- a/pkg/api/go.mod +++ b/pkg/api/go.mod @@ -76,7 +76,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/pkg/api/go.sum b/pkg/api/go.sum index 9694f5d5e2355..dd16364891695 100644 --- a/pkg/api/go.sum +++ b/pkg/api/go.sum @@ -87,8 +87,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/api/security/security.go b/pkg/api/security/security.go index 40b1310c2f79c..93867572572d0 100644 --- a/pkg/api/security/security.go +++ b/pkg/api/security/security.go @@ -239,7 +239,7 @@ func validateAuthToken(authToken string) error { // writes auth token(s) to a file with the same permissions as datadog.yaml func saveAuthToken(token, tokenPath string) error { log.Infof("Saving a new authentication token in %s", tokenPath) - if err := os.WriteFile(tokenPath, []byte(token), 0o600); err != nil { + if err := writeFileAndSync(tokenPath, []byte(token), 0o600); err != nil { return err } @@ -256,3 +256,21 @@ func saveAuthToken(token, tokenPath string) error { log.Infof("Wrote auth token in %s", tokenPath) return nil } + +// call the file.Sync method to flush the file to disk and catch potential I/O errors +func writeFileAndSync(name string, data []byte, perm os.FileMode) error { + f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = f.Write(data) + // only sync data if the write was successful + if err == nil { + err = f.Sync() + } + // but always close the file and report the first error that occurred + if err1 := f.Close(); err1 != nil && err == nil { + err = err1 + } + return err +} diff --git a/pkg/clusteragent/admission/controllers/webhook/config.go b/pkg/clusteragent/admission/controllers/webhook/config.go index 90cec93a8d8db..4f5e549434557 100644 --- a/pkg/clusteragent/admission/controllers/webhook/config.go +++ b/pkg/clusteragent/admission/controllers/webhook/config.go @@ -13,7 +13,7 @@ import ( "fmt" "strings" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/common" ) @@ -36,21 +36,21 @@ type Config struct { } // NewConfig creates a webhook controller configuration -func NewConfig(admissionV1Enabled, namespaceSelectorEnabled, matchConditionsSupported bool) Config { +func NewConfig(admissionV1Enabled, namespaceSelectorEnabled, matchConditionsSupported bool, datadogConfig config.Component) Config { return Config{ - webhookName: pkgconfigsetup.Datadog().GetString("admission_controller.webhook_name"), - secretName: pkgconfigsetup.Datadog().GetString("admission_controller.certificate.secret_name"), - validationEnabled: pkgconfigsetup.Datadog().GetBool("admission_controller.validation.enabled"), - mutationEnabled: pkgconfigsetup.Datadog().GetBool("admission_controller.mutation.enabled"), + webhookName: datadogConfig.GetString("admission_controller.webhook_name"), + secretName: datadogConfig.GetString("admission_controller.certificate.secret_name"), + validationEnabled: datadogConfig.GetBool("admission_controller.validation.enabled"), + mutationEnabled: datadogConfig.GetBool("admission_controller.mutation.enabled"), namespace: common.GetResourcesNamespace(), admissionV1Enabled: admissionV1Enabled, namespaceSelectorEnabled: namespaceSelectorEnabled, matchConditionsSupported: matchConditionsSupported, - svcName: pkgconfigsetup.Datadog().GetString("admission_controller.service_name"), + svcName: datadogConfig.GetString("admission_controller.service_name"), svcPort: int32(443), - timeout: pkgconfigsetup.Datadog().GetInt32("admission_controller.timeout_seconds"), - failurePolicy: pkgconfigsetup.Datadog().GetString("admission_controller.failure_policy"), - reinvocationPolicy: pkgconfigsetup.Datadog().GetString("admission_controller.reinvocation_policy"), + timeout: datadogConfig.GetInt32("admission_controller.timeout_seconds"), + failurePolicy: datadogConfig.GetString("admission_controller.failure_policy"), + reinvocationPolicy: datadogConfig.GetString("admission_controller.reinvocation_policy"), } } diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_base.go b/pkg/clusteragent/admission/controllers/webhook/controller_base.go index 79114dac7397f..5505337ba5838 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_base.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_base.go @@ -35,7 +35,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate/tagsfromlabels" "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/validate/kubernetesadmissionevents" "github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -125,7 +124,7 @@ func (c *controllerBase) generateWebhooks(wmeta workloadmeta.Component, pa workl configWebhook.NewWebhook(wmeta, injectionFilter, datadogConfig), tagsfromlabels.NewWebhook(wmeta, datadogConfig, injectionFilter), agentsidecar.NewWebhook(datadogConfig), - autoscaling.NewWebhook(pa), + autoscaling.NewWebhook(pa, datadogConfig), } webhooks = append(webhooks, mutatingWebhooks...) @@ -137,7 +136,7 @@ func (c *controllerBase) generateWebhooks(wmeta workloadmeta.Component, pa workl log.Errorf("failed to register APM Instrumentation webhook: %v", err) } - isCWSInstrumentationEnabled := pkgconfigsetup.Datadog().GetBool("admission_controller.cws_instrumentation.enabled") + isCWSInstrumentationEnabled := datadogConfig.GetBool("admission_controller.cws_instrumentation.enabled") if isCWSInstrumentationEnabled { cws, err := cwsinstrumentation.NewCWSInstrumentation(wmeta, datadogConfig) if err == nil { diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go index c202f1ab38f25..d5612b96c74e0 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go @@ -36,7 +36,7 @@ func TestNewController(t *testing.T) { factory.Admissionregistration(), func() bool { return true }, make(chan struct{}), - v1Cfg, + getV1Cfg(t), wmeta, nil, datadogConfig, @@ -53,7 +53,7 @@ func TestNewController(t *testing.T) { factory.Admissionregistration(), func() bool { return true }, make(chan struct{}), - v1beta1Cfg, + getV1beta1Cfg(t), wmeta, nil, datadogConfig, diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go index aa2a3efc4e68e..6a52255c42e29 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go @@ -41,10 +41,11 @@ const ( tick = 50 * time.Millisecond ) -var v1Cfg = NewConfig(true, false, false) +func getV1Cfg(t *testing.T) Config { return NewConfig(true, false, false, configComp.NewMock(t)) } func TestSecretNotFoundV1(t *testing.T) { f := newFixtureV1(t) + v1Cfg := getV1Cfg(t) stopCh := make(chan struct{}) defer close(stopCh) @@ -63,6 +64,7 @@ func TestSecretNotFoundV1(t *testing.T) { func TestCreateWebhookV1(t *testing.T) { f := newFixtureV1(t) + v1Cfg := getV1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -97,6 +99,7 @@ func TestCreateWebhookV1(t *testing.T) { func TestUpdateOutdatedWebhookV1(t *testing.T) { f := newFixtureV1(t) + v1Cfg := getV1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -166,7 +169,7 @@ func TestAdmissionControllerFailureModeV1(t *testing.T) { for _, value := range []string{"Ignore", "ignore", "BadVal", ""} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1.OperationType{admiv1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1.Ignore, *validatingWebhookSkeleton.FailurePolicy) @@ -176,7 +179,7 @@ func TestAdmissionControllerFailureModeV1(t *testing.T) { for _, value := range []string{"Fail", "fail"} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1.OperationType{admiv1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1.Fail, *validatingWebhookSkeleton.FailurePolicy) @@ -192,7 +195,7 @@ func TestAdmissionControllerReinvocationPolicyV1(t *testing.T) { for _, value := range []string{"IfNeeded", "ifneeded", "Never", "never", "wrong", ""} { mockConfig.SetWithoutSource("admission_controller.reinvocationpolicy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) mutatingWebhookSkeleton := c.getMutatingWebhookSkeleton("foo", "/bar", []admiv1.OperationType{admiv1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1.IfNeededReinvocationPolicy, *mutatingWebhookSkeleton.ReinvocationPolicy) @@ -241,7 +244,7 @@ func TestGenerateTemplatesV1(t *testing.T) { tests := []struct { name string setupConfig func(model.Config) - configFunc func() Config + configFunc func(model.Config) Config want func() []admiv1.MutatingWebhook }{ { @@ -253,7 +256,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -276,7 +279,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -295,7 +298,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -318,7 +321,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -337,7 +340,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -360,7 +363,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -378,7 +381,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -402,7 +405,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -435,7 +438,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", nil, &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -461,7 +464,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -511,7 +514,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -559,7 +562,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -602,7 +605,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -641,7 +644,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -684,7 +687,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -726,7 +729,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "misconfigured") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -746,7 +749,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -779,7 +782,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -799,7 +802,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -819,7 +822,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -850,7 +853,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -870,7 +873,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -899,7 +902,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey1\": \"labelVal1\"}}, \"NamespaceSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -928,7 +931,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\":{\"labelKey1\": \"labelVal1\"}}} , {\"ObjectSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -947,7 +950,7 @@ func TestGenerateTemplatesV1(t *testing.T) { tt.setupConfig(mockConfig) c := &ControllerV1{} - c.config = tt.configFunc() + c.config = tt.configFunc(mockConfig) c.webhooks = c.generateWebhooks(wmeta, nil, mockConfig, nil) c.generateTemplates() @@ -1051,7 +1054,7 @@ func TestGetValidatingWebhookSkeletonV1(t *testing.T) { } c := &ControllerV1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1157,7 +1160,7 @@ func TestGetMutatingWebhookSkeletonV1(t *testing.T) { } c := &ControllerV1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1188,7 +1191,7 @@ func (f *fixtureV1) createController() (*ControllerV1, informers.SharedInformerF factory.Admissionregistration().V1().MutatingWebhookConfigurations(), func() bool { return true }, make(chan struct{}), - v1Cfg, + getV1Cfg(f.t), wmeta, nil, datadogConfig, diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go index bd5daf906a2fb..62fa7713cd809 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go @@ -36,10 +36,11 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/kubernetes/certificate" ) -var v1beta1Cfg = NewConfig(false, false, false) +func getV1beta1Cfg(t *testing.T) Config { return NewConfig(false, false, false, configComp.NewMock(t)) } func TestSecretNotFoundV1beta1(t *testing.T) { f := newFixtureV1beta1(t) + v1beta1Cfg := getV1beta1Cfg(t) stopCh := make(chan struct{}) defer close(stopCh) @@ -58,6 +59,7 @@ func TestSecretNotFoundV1beta1(t *testing.T) { func TestCreateWebhookV1beta1(t *testing.T) { f := newFixtureV1beta1(t) + v1beta1Cfg := getV1beta1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -92,6 +94,8 @@ func TestCreateWebhookV1beta1(t *testing.T) { func TestUpdateOutdatedWebhookV1beta1(t *testing.T) { f := newFixtureV1beta1(t) + v1Cfg := getV1Cfg(t) + v1beta1Cfg := getV1beta1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -161,7 +165,7 @@ func TestAdmissionControllerFailureModeV1beta1(t *testing.T) { for _, value := range []string{"Ignore", "ignore", "BadVal", ""} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1beta1.OperationType{admiv1beta1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1beta1.Ignore, *validatingWebhookSkeleton.FailurePolicy) @@ -171,7 +175,7 @@ func TestAdmissionControllerFailureModeV1beta1(t *testing.T) { for _, value := range []string{"Fail", "fail"} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1beta1.OperationType{admiv1beta1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1beta1.Fail, *validatingWebhookSkeleton.FailurePolicy) @@ -187,7 +191,7 @@ func TestAdmissionControllerReinvocationPolicyV1beta1(t *testing.T) { for _, value := range []string{"IfNeeded", "ifneeded", "Never", "never", "wrong", ""} { mockConfig.SetWithoutSource("admission_controller.reinvocationpolicy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) mutatingWebhookSkeleton := c.getMutatingWebhookSkeleton("foo", "/bar", []admiv1beta1.OperationType{admiv1beta1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1beta1.IfNeededReinvocationPolicy, *mutatingWebhookSkeleton.ReinvocationPolicy) @@ -236,7 +240,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { tests := []struct { name string setupConfig func(model.Config) - configFunc func() Config + configFunc func(model.Config) Config want func() []admiv1beta1.MutatingWebhook }{ { @@ -248,7 +252,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -271,7 +275,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -290,7 +294,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -313,7 +317,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -332,7 +336,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -355,7 +359,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -373,7 +377,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -397,7 +401,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -430,7 +434,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", nil, &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -456,7 +460,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -505,7 +509,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -552,7 +556,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -595,7 +599,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -634,7 +638,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -677,7 +681,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -719,7 +723,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "misconfigured") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -739,7 +743,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -772,7 +776,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -792,7 +796,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -812,7 +816,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -843,7 +847,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -863,7 +867,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -892,7 +896,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey1\": \"labelVal1\"}}, \"NamespaceSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -921,7 +925,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\":{\"labelKey1\": \"labelVal1\"}}} , {\"ObjectSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -940,7 +944,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { tt.setupConfig(mockConfig) c := &ControllerV1beta1{} - c.config = tt.configFunc() + c.config = tt.configFunc(mockConfig) c.webhooks = c.generateWebhooks(wmeta, nil, mockConfig, nil) c.generateTemplates() @@ -1044,7 +1048,7 @@ func TestGetValidatingWebhookSkeletonV1beta1(t *testing.T) { } c := &ControllerV1beta1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1150,7 +1154,7 @@ func TestGetMutatingWebhookSkeletonV1beta1(t *testing.T) { } c := &ControllerV1beta1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1181,7 +1185,7 @@ func (f *fixtureV1beta1) createController() (*ControllerV1beta1, informers.Share factory.Admissionregistration().V1beta1().MutatingWebhookConfigurations(), func() bool { return true }, make(chan struct{}), - v1beta1Cfg, + getV1beta1Cfg(f.t), wmeta, nil, datadogConfig, diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go index 06c936d6d0684..302fa2f36b0e0 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go @@ -34,6 +34,7 @@ import ( apiCommon "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/common" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername" "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) const webhookName = "agent_sidecar" @@ -59,7 +60,7 @@ type Webhook struct { // These fields store datadog agent config parameters // to avoid calling the config resolution each time the webhook // receives requests because the resolution is CPU expensive. - profilesJSON string + profileOverrides []ProfileOverride provider string imageName string imageTag string @@ -72,7 +73,12 @@ type Webhook struct { // NewWebhook returns a new Webhook func NewWebhook(datadogConfig config.Component) *Webhook { - nsSelector, objSelector := labelSelectors(datadogConfig) + profileOverrides, err := loadSidecarProfiles(datadogConfig.GetString("admission_controller.agent_sidecar.profiles")) + if err != nil { + log.Errorf("encountered issue when loading sidecar profiles: %s", err) + } + + nsSelector, objSelector := labelSelectors(datadogConfig, profileOverrides) containerRegistry := mutatecommon.ContainerRegistry(datadogConfig, "admission_controller.agent_sidecar.container_registry") @@ -86,8 +92,8 @@ func NewWebhook(datadogConfig config.Component) *Webhook { namespaceSelector: nsSelector, objectSelector: objSelector, containerRegistry: containerRegistry, + profileOverrides: profileOverrides, - profilesJSON: datadogConfig.GetString("admission_controller.agent_sidecar.profiles"), provider: datadogConfig.GetString("admission_controller.agent_sidecar.provider"), imageName: datadogConfig.GetString("admission_controller.agent_sidecar.image_name"), imageTag: datadogConfig.GetString("admission_controller.agent_sidecar.image_tag"), @@ -150,6 +156,18 @@ func (w *Webhook) WebhookFunc() admission.WebhookFunc { } } +// isReadOnlyRootFilesystem returns whether the agent sidecar should have the readOnlyRootFilesystem security setup +func (w *Webhook) isReadOnlyRootFilesystem() bool { + if len(w.profileOverrides) == 0 { + return false + } + securityContext := w.profileOverrides[0].SecurityContext + if securityContext != nil && securityContext.ReadOnlyRootFilesystem != nil { + return *securityContext.ReadOnlyRootFilesystem + } + return false // default to false (temp) +} + func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interface) (bool, error) { if pod == nil { return false, errors.New(metrics.InvalidInput) @@ -163,6 +181,16 @@ func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interf if !agentSidecarExists { agentSidecarContainer := w.getDefaultSidecarTemplate() + if w.isReadOnlyRootFilesystem() { + volumes := w.getSecurityVolumeTemplates() + pod.Spec.Volumes = append(pod.Spec.Volumes, *volumes...) + w.addSecurityConfigToAgent(agentSidecarContainer) + // Don't want to apply any overrides to the agent sidecar init container + defer func() { + initContainer := w.getSecurityInitTemplate() + pod.Spec.InitContainers = append(pod.Spec.InitContainers, *initContainer) + }() + } pod.Spec.Containers = append(pod.Spec.Containers, *agentSidecarContainer) podUpdated = true } @@ -190,7 +218,7 @@ func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interf } podUpdated = podUpdated || updated - updated, err = applyProfileOverrides(&pod.Spec.Containers[i], w.profilesJSON) + updated, err = applyProfileOverrides(&pod.Spec.Containers[i], w.profileOverrides) if err != nil { log.Errorf("Failed to apply profile overrides: %v", err) return podUpdated, errors.New(metrics.InvalidInput) @@ -204,6 +232,77 @@ func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interf return podUpdated, nil } +func (w *Webhook) getSecurityInitTemplate() *corev1.Container { + return &corev1.Container{ + Image: fmt.Sprintf("%s/%s:%s", w.containerRegistry, w.imageName, w.imageTag), + ImagePullPolicy: corev1.PullIfNotPresent, + Name: "init-copy-agent-config", + Command: []string{"sh", "-c", "cp -R /etc/datadog-agent/* /agent-config/"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: agentConfigVolumeName, + MountPath: "/agent-config", + }, + }, + } +} + +func (w *Webhook) getSecurityVolumeTemplates() *[]corev1.Volume { + return &[]corev1.Volume{ + { + Name: agentConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: agentOptionsVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: agentTmpVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: agentLogsVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } +} + +func (w *Webhook) addSecurityConfigToAgent(agentContainer *corev1.Container) { + volumeMounts := []corev1.VolumeMount{ + { + Name: agentConfigVolumeName, + MountPath: "/etc/datadog-agent", + }, + { + Name: agentOptionsVolumeName, + MountPath: "/opt/datadog-agent/run", + }, + { + Name: agentTmpVolumeName, + MountPath: "/tmp", + }, + { + Name: agentLogsVolumeName, + MountPath: "/var/log/datadog", + }, + } + agentContainer.VolumeMounts = append(agentContainer.VolumeMounts, volumeMounts...) + + if agentContainer.SecurityContext == nil { + agentContainer.SecurityContext = &corev1.SecurityContext{} + } + agentContainer.SecurityContext.ReadOnlyRootFilesystem = pointer.Ptr(true) +} + func (w *Webhook) getDefaultSidecarTemplate() *corev1.Container { ddSite := os.Getenv("DD_SITE") if ddSite == "" { @@ -288,21 +387,19 @@ func (w *Webhook) getDefaultSidecarTemplate() *corev1.Container { } // labelSelectors returns the mutating webhooks object selectors based on the configuration -func labelSelectors(datadogConfig config.Component) (namespaceSelector, objectSelector *metav1.LabelSelector) { +func labelSelectors(datadogConfig config.Component, profileOverrides []ProfileOverride) (namespaceSelector, objectSelector *metav1.LabelSelector) { // Read and parse selectors selectorsJSON := datadogConfig.GetString("admission_controller.agent_sidecar.selectors") - profilesJSON := datadogConfig.GetString("admission_controller.agent_sidecar.profiles") // Get sidecar profiles - _, err := loadSidecarProfiles(profilesJSON) - if err != nil { - log.Errorf("encountered issue when loading sidecar profiles: %s", err) + if profileOverrides == nil { + log.Error("sidecar profiles are not loaded") return nil, nil } var selectors []Selector - err = json.Unmarshal([]byte(selectorsJSON), &selectors) + err := json.Unmarshal([]byte(selectorsJSON), &selectors) if err != nil { log.Errorf("failed to parse selectors for admission controller agent sidecar injection webhook: %s", err) return nil, nil diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go index aac87fce23cba..9d6dd44be2d90 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go @@ -59,19 +59,67 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - provider: "", - profilesJSON: "[]", + provider: "", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + webhook.addSecurityConfigToAgent(sidecar) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name", }, Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{*webhook.getSecurityInitTemplate()}, Containers: []corev1.Container{ {Name: "container-name"}, - *NewWebhook(mockConfig).getDefaultSidecarTemplate(), + *sidecar, + }, + Volumes: *webhook.getSecurityVolumeTemplates(), + }, + } + }, + }, + { + Name: "should inject sidecar, no security features if default overridden to false", + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + }, + }, + }, + provider: "", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": false + } + }]`, + ExpectError: false, + ExpectInjection: true, + ExpectedPodAfterInjection: func() *corev1.Pod { + sidecar := *NewWebhook(mockConfig).getDefaultSidecarTemplate() + // Records the false readOnlyRootFilesystem but doesn't add the initContainers, volumes and mounts + sidecar.SecurityContext = &corev1.SecurityContext{ + ReadOnlyRootFilesystem: pointer.Ptr(false), + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + sidecar, }, }, } @@ -157,8 +205,12 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - provider: "", - profilesJSON: "[]", + provider: "", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": false + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { @@ -168,6 +220,9 @@ func TestInjectAgentSidecar(t *testing.T) { Name: "DD_AUTO_EXIT_NOPROCESS_ENABLED", Value: "true", }) + defaultContainer.SecurityContext = &corev1.SecurityContext{ + ReadOnlyRootFilesystem: pointer.Ptr(false), + } return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name", @@ -199,14 +254,20 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - provider: "fargate", - profilesJSON: "[]", + provider: "fargate", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { - sidecar := *NewWebhook(mockConfig).getDefaultSidecarTemplate() + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + webhook.addSecurityConfigToAgent(sidecar) _, _ = withEnvOverrides( - &sidecar, + sidecar, corev1.EnvVar{ Name: "DD_EKS_FARGATE", Value: "true", @@ -221,13 +282,13 @@ func TestInjectAgentSidecar(t *testing.T) { }, ) - sidecar.VolumeMounts = []corev1.VolumeMount{ + sidecar.VolumeMounts = append(sidecar.VolumeMounts, []corev1.VolumeMount{ { Name: "ddsockets", MountPath: "/var/run/datadog", ReadOnly: false, }, - } + }...) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -238,6 +299,7 @@ func TestInjectAgentSidecar(t *testing.T) { }, Spec: corev1.PodSpec{ ShareProcessNamespace: pointer.Ptr(true), + InitContainers: []corev1.Container{*webhook.getSecurityInitTemplate()}, Containers: []corev1.Container{ { Name: "container-name", @@ -259,16 +321,16 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - sidecar, + *sidecar, }, - Volumes: []corev1.Volume{ - { + Volumes: append(*webhook.getSecurityVolumeTemplates(), + corev1.Volume{ Name: "ddsockets", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - }, + ), }, } }, @@ -287,28 +349,33 @@ func TestInjectAgentSidecar(t *testing.T) { }, provider: "fargate", profilesJSON: `[{ - "env": [ - {"name": "ENV_VAR_1", "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "secret-key"}}}, - {"name": "ENV_VAR_2", "value": "value2"} - ], - "resources": { - "limits": { - "cpu": "1", - "memory": "512Mi" - }, - "requests": { - "cpu": "0.5", - "memory": "256Mi" - } - } - }]`, + "env": [ + {"name": "ENV_VAR_1", "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "secret-key"}}}, + {"name": "ENV_VAR_2", "value": "value2"} + ], + "resources": { + "limits": { + "cpu": "1", + "memory": "512Mi" + }, + "requests": { + "cpu": "0.5", + "memory": "256Mi" + } + }, + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { - sidecar := *NewWebhook(mockConfig).getDefaultSidecarTemplate() + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + webhook.addSecurityConfigToAgent(sidecar) _, _ = withEnvOverrides( - &sidecar, + sidecar, corev1.EnvVar{ Name: "DD_EKS_FARGATE", Value: "true", @@ -336,18 +403,18 @@ func TestInjectAgentSidecar(t *testing.T) { }, ) - _ = withResourceLimits(&sidecar, corev1.ResourceRequirements{ + _ = withResourceLimits(sidecar, corev1.ResourceRequirements{ Limits: corev1.ResourceList{"cpu": resource.MustParse("1"), "memory": resource.MustParse("512Mi")}, Requests: corev1.ResourceList{"cpu": resource.MustParse("0.5"), "memory": resource.MustParse("256Mi")}, }) - sidecar.VolumeMounts = []corev1.VolumeMount{ + sidecar.VolumeMounts = append(sidecar.VolumeMounts, []corev1.VolumeMount{ { Name: "ddsockets", MountPath: "/var/run/datadog", ReadOnly: false, }, - } + }...) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -358,6 +425,7 @@ func TestInjectAgentSidecar(t *testing.T) { }, Spec: corev1.PodSpec{ ShareProcessNamespace: pointer.Ptr(true), + InitContainers: []corev1.Container{*webhook.getSecurityInitTemplate()}, Containers: []corev1.Container{ { Name: "container-name", @@ -379,16 +447,16 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - sidecar, + *sidecar, }, - Volumes: []corev1.Volume{ - { + Volumes: append(*webhook.getSecurityVolumeTemplates(), + corev1.Volume{ Name: "ddsockets", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - }, + ), }, } }, @@ -644,3 +712,80 @@ func TestDefaultSidecarTemplateClusterAgentEnvVars(t *testing.T) { }) } } + +func TestIsReadOnlyRootFilesystem(t *testing.T) { + tests := []struct { + name string + profile string + expected bool + }{ + { + name: "no profile", + profile: "", + expected: false, + }, + { + name: "empty or default profile", + profile: "[]", + expected: false, + }, + { + name: "profile without security context", + profile: `[{ + "env": [ + {"name": "ENV_VAR_2", "value": "value2"} + ], + }]`, + expected: false, + }, + { + name: "profile with security context, readOnlyRootFilesystem empty", + profile: `[{ + "securityContext": {} + }]`, + expected: false, + }, + { + name: "profile with security context, readOnlyRootFilesystem true", + profile: `[{ + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, + expected: true, + }, + { + name: "profile with security context, readOnlyRootFilesystem false", + profile: `[{ + "securityContext": { + "readOnlyRootFilesystem": false + } + }]`, + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + mockConfig := configmock.New(t) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", test.profile) + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + + // Webhook properly parses profile config + assert.Equal(tt, test.expected, webhook.isReadOnlyRootFilesystem()) + + if test.expected { + // Webhook properly applies the security context to the sidecar + webhook.addSecurityConfigToAgent(sidecar) + assert.NotNil(t, sidecar.SecurityContext) + assert.NotNil(t, sidecar.SecurityContext.ReadOnlyRootFilesystem) + assert.Equal(t, test.expected, *sidecar.SecurityContext.ReadOnlyRootFilesystem) + } else { + assert.Nil(t, sidecar.SecurityContext) + profile, _ := loadSidecarProfiles(test.profile) + applyProfileOverrides(sidecar, profile) + } + }) + } +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go b/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go index fd73e16272e54..dfd44618d3206 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go @@ -11,3 +11,10 @@ const ( agentSidecarContainerName = "datadog-agent-injected" providerFargate = "fargate" ) + +const ( + agentConfigVolumeName = "agent-config" + agentOptionsVolumeName = "agent-option" + agentTmpVolumeName = "agent-tmp" + agentLogsVolumeName = "agent-log" +) diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go index 2a42c1d8947f1..9389d381a6c01 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go @@ -54,3 +54,19 @@ func withResourceLimits(container *corev1.Container, resourceLimits corev1.Resou container.Resources = resourceLimits return nil } + +// withSecurityContextOverrides applies the security context overrides to the container +func withSecurityContextOverrides(container *corev1.Container, securityContext *corev1.SecurityContext) (bool, error) { + if container == nil { + return false, fmt.Errorf("can't apply security context overrides to nil container") + } + + mutated := false + + if securityContext != nil { + container.SecurityContext = securityContext + mutated = true + } + + return mutated, nil +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go index bde80aea94ab2..13ab63c9fd77e 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func TestWithEnvOverrides(t *testing.T) { @@ -144,3 +146,92 @@ func TestWithResourceLimits(t *testing.T) { }) } } + +func TestWithSecurityContextOverrides(t *testing.T) { + tests := []struct { + name string + baseContainer *corev1.Container + securityOverride *corev1.SecurityContext + expectError bool + expectMutated bool + containerAfterOverride *corev1.Container + }{ + { + name: "nil container", + baseContainer: nil, + securityOverride: &corev1.SecurityContext{}, + expectError: true, + expectMutated: false, + containerAfterOverride: nil, + }, + { + name: "no overrides", + baseContainer: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + securityOverride: nil, + expectError: false, + expectMutated: false, + containerAfterOverride: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + }, + { + name: "apply overrides", + baseContainer: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + securityOverride: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(false), + }, + expectError: false, + expectMutated: true, + containerAfterOverride: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(false), + }, + }, + }, + { + name: "apply blank overrides", + baseContainer: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + securityOverride: &corev1.SecurityContext{}, + expectError: false, + expectMutated: true, + containerAfterOverride: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + mutated, err := withSecurityContextOverrides(test.baseContainer, test.securityOverride) + + assert.Equal(tt, test.expectMutated, mutated) + assert.Equal(tt, test.containerAfterOverride, test.baseContainer) + + if test.expectError { + assert.Error(tt, err) + } else { + assert.NoError(tt, err) + } + + }) + } +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go index 101f54bd059eb..79430f8d8cbca 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go @@ -25,6 +25,7 @@ import ( type ProfileOverride struct { EnvVars []corev1.EnvVar `json:"env,omitempty"` ResourceRequirements corev1.ResourceRequirements `json:"resources,omitempty"` + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` } // loadSidecarProfiles returns the profile overrides provided by the user @@ -48,16 +49,15 @@ func loadSidecarProfiles(profilesJSON string) ([]ProfileOverride, error) { // applyProfileOverrides applies the profile overrides to the container. It // returns a boolean that indicates if the container was mutated -func applyProfileOverrides(container *corev1.Container, profilesJSON string) (bool, error) { +func applyProfileOverrides(container *corev1.Container, profiles []ProfileOverride) (bool, error) { if container == nil { return false, fmt.Errorf("can't apply profile overrides to nil containers") } - profiles, err := loadSidecarProfiles(profilesJSON) - - if err != nil { - return false, err + if profiles == nil { + return false, fmt.Errorf("can't apply nil profiles") } + if len(profiles) == 0 { return false, nil } @@ -82,5 +82,12 @@ func applyProfileOverrides(container *corev1.Container, profilesJSON string) (bo mutated = true } + // Apply security context overrides + overridden, err = withSecurityContextOverrides(container, overrides.SecurityContext) + if err != nil { + return false, err + } + mutated = mutated || overridden + return mutated, nil } diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go index 772e8b784a78d..4b8c007d47fe5 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func TestLoadSidecarProfiles(t *testing.T) { @@ -29,6 +31,12 @@ func TestLoadSidecarProfiles(t *testing.T) { expectedProfiles: nil, expectError: true, }, + { + name: "default profile", + profilesJSON: "[]", + expectedProfiles: []ProfileOverride{}, + expectError: false, + }, { name: "single valid profile", profilesJSON: `[ @@ -46,6 +54,9 @@ func TestLoadSidecarProfiles(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } + }, + "securityContext": { + "readOnlyRootFilesystem": true } } ]`, @@ -65,6 +76,7 @@ func TestLoadSidecarProfiles(t *testing.T) { "memory": resource.MustParse("256Mi"), }, }, + SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: pointer.Ptr(true)}, }, }, expectError: false, @@ -166,6 +178,11 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } + }, + "securityContext": { + "readOnlyRootFilesystem": true, + "runAsNonRoot": false, + "runAsUser": 1000 } } ]`, @@ -185,6 +202,7 @@ func TestApplyProfileOverrides(t *testing.T) { "memory": resource.MustParse("256Mi"), }, }, + SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: pointer.Ptr(true), RunAsNonRoot: pointer.Ptr(false), RunAsUser: pointer.Ptr(int64(1000))}, }, expectError: false, expectMutated: true, @@ -247,7 +265,10 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } - } + }, + "securityContext": { + "readOnlyRootFilesystem": true + } }]`, baseContainer: &corev1.Container{}, expectedContainer: &corev1.Container{ @@ -261,6 +282,7 @@ func TestApplyProfileOverrides(t *testing.T) { Limits: corev1.ResourceList{"cpu": resource.MustParse("1"), "memory": resource.MustParse("512Mi")}, Requests: corev1.ResourceList{"cpu": resource.MustParse("0.5"), "memory": resource.MustParse("256Mi")}, }, + SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: pointer.Ptr(true)}, }, expectError: false, expectMutated: true, @@ -326,7 +348,10 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } - } + }, + "securityContext": { + "readOnlyRootFilesystem": true + } }, { "env": [ @@ -341,7 +366,10 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "1", "memory": "512Mi" } - } + }, + "securityContext": { + "readOnlyRootFilesystem": false + } } ]`, baseContainer: &corev1.Container{}, @@ -353,7 +381,8 @@ func TestApplyProfileOverrides(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - mutated, err := applyProfileOverrides(test.baseContainer, test.profilesJSON) + profiles, _ := loadSidecarProfiles(test.profilesJSON) + mutated, err := applyProfileOverrides(test.baseContainer, profiles) assert.Equal(tt, test.expectMutated, mutated) diff --git a/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go b/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go index b4e7225ef710e..4ac5396506357 100644 --- a/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go +++ b/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go @@ -16,10 +16,10 @@ import ( "k8s.io/client-go/dynamic" "github.com/DataDog/datadog-agent/cmd/cluster-agent/admission" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/common" mutatecommon "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate/common" "github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" ) const ( @@ -39,10 +39,10 @@ type Webhook struct { } // NewWebhook returns a new Webhook -func NewWebhook(patcher workload.PodPatcher) *Webhook { +func NewWebhook(patcher workload.PodPatcher, datadogConfig config.Component) *Webhook { return &Webhook{ name: webhookName, - isEnabled: pkgconfigsetup.Datadog().GetBool("autoscaling.workload.enabled"), + isEnabled: datadogConfig.GetBool("autoscaling.workload.enabled"), endpoint: webhookEndpoint, resources: map[string][]string{"": {"pods"}}, operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, diff --git a/pkg/clusteragent/admission/start.go b/pkg/clusteragent/admission/start.go index aee1183a3133d..ef69edfd4a203 100644 --- a/pkg/clusteragent/admission/start.go +++ b/pkg/clusteragent/admission/start.go @@ -79,7 +79,7 @@ func StartControllers(ctx ControllerContext, wmeta workloadmeta.Component, pa wo return webhooks, err } - webhookConfig := webhook.NewConfig(v1Enabled, nsSelectorEnabled, matchConditionsSupported) + webhookConfig := webhook.NewConfig(v1Enabled, nsSelectorEnabled, matchConditionsSupported, datadogConfig) webhookController := webhook.NewController( ctx.Client, ctx.SecretInformers.Core().V1().Secrets(), diff --git a/pkg/clusteragent/autoscaling/workload/controller_horizontal.go b/pkg/clusteragent/autoscaling/workload/controller_horizontal.go index 555cf50b181d3..5bd044a18885c 100644 --- a/pkg/clusteragent/autoscaling/workload/controller_horizontal.go +++ b/pkg/clusteragent/autoscaling/workload/controller_horizontal.go @@ -246,6 +246,27 @@ func (hr *horizontalController) computeScaleAction( evalAfter = rulesNextEvalAfter.Truncate(time.Second) + time.Second } + // Stabilize recommendation + var stabilizationLimitReason string + var stabilizationLimitedReplicas int32 + upscaleStabilizationSeconds := int32(0) + downscaleStabilizationSeconds := int32(0) + + if policy := autoscalerInternal.Spec().Policy; policy != nil { + if upscalePolicy := policy.Upscale; upscalePolicy != nil { + upscaleStabilizationSeconds = int32(upscalePolicy.StabilizationWindowSeconds) + } + if downscalePolicy := policy.Downscale; downscalePolicy != nil { + downscaleStabilizationSeconds = int32(downscalePolicy.StabilizationWindowSeconds) + } + } + + stabilizationLimitedReplicas, stabilizationLimitReason = stabilizeRecommendations(scalingTimestamp, autoscalerInternal.HorizontalLastActions(), currentDesiredReplicas, targetDesiredReplicas, upscaleStabilizationSeconds, downscaleStabilizationSeconds, scaleDirection) + if stabilizationLimitReason != "" { + limitReason = stabilizationLimitReason + targetDesiredReplicas = stabilizationLimitedReplicas + } + horizontalAction := &datadoghq.DatadogPodAutoscalerHorizontalAction{ FromReplicas: currentDesiredReplicas, ToReplicas: targetDesiredReplicas, @@ -457,3 +478,44 @@ func accumulateReplicasChange(currentTime time.Time, events []datadoghq.DatadogP } return } + +func stabilizeRecommendations(currentTime time.Time, pastActions []datadoghq.DatadogPodAutoscalerHorizontalAction, currentReplicas int32, originalTargetDesiredReplicas int32, stabilizationWindowUpscaleSeconds int32, stabilizationWindowDownscaleSeconds int32, scaleDirection scaleDirection) (int32, string) { + limitReason := "" + + if len(pastActions) == 0 { + return originalTargetDesiredReplicas, limitReason + } + + upRecommendation := originalTargetDesiredReplicas + upCutoff := currentTime.Add(-time.Duration(stabilizationWindowUpscaleSeconds) * time.Second) + + downRecommendation := originalTargetDesiredReplicas + downCutoff := currentTime.Add(-time.Duration(stabilizationWindowDownscaleSeconds) * time.Second) + + for _, a := range pastActions { + if scaleDirection == scaleUp && a.Time.Time.After(upCutoff) { + upRecommendation = min(upRecommendation, *a.RecommendedReplicas) + } + + if scaleDirection == scaleDown && a.Time.Time.After(downCutoff) { + downRecommendation = max(downRecommendation, *a.RecommendedReplicas) + } + + if (scaleDirection == scaleUp && a.Time.Time.Before(upCutoff)) || (scaleDirection == scaleDown && a.Time.Time.Before(downCutoff)) { + break + } + } + + recommendation := currentReplicas + if recommendation < upRecommendation { + recommendation = upRecommendation + } + if recommendation > downRecommendation { + recommendation = downRecommendation + } + if recommendation != originalTargetDesiredReplicas { + limitReason = fmt.Sprintf("desired replica count limited to %d (originally %d) due to stabilization window", recommendation, originalTargetDesiredReplicas) + } + + return recommendation, limitReason +} diff --git a/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go b/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go index fc2eb1b1fe88c..89014b196a066 100644 --- a/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go +++ b/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go @@ -77,6 +77,15 @@ func (f *horizontalControllerFixture) runSync(fakePai *model.FakePodAutoscalerIn return autoscalerInternal, res, err } +func newHorizontalAction(time time.Time, fromReplicas, toReplicas, recommendedReplicas int32) datadoghq.DatadogPodAutoscalerHorizontalAction { + return datadoghq.DatadogPodAutoscalerHorizontalAction{ + Time: metav1.NewTime(time), + FromReplicas: fromReplicas, + ToReplicas: toReplicas, + RecommendedReplicas: pointer.Ptr[int32](recommendedReplicas), + } +} + type horizontalScalingTestArgs struct { fakePai *model.FakePodAutoscalerInternal dataSource datadoghq.DatadogPodAutoscalerValueSource @@ -181,6 +190,14 @@ func TestHorizontalControllerSyncPrerequisites(t *testing.T) { Kind: expectedGVK.Kind, APIVersion: expectedGVK.Group + "/" + expectedGVK.Version, }, + Policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + }, } fakePai.Error = nil f.scaler.On("get", mock.Anything, autoscalerNamespace, autoscalerName, expectedGVK).Return( @@ -1119,3 +1136,352 @@ func TestHorizontalControllerSyncScaleDecisionsWithRules(t *testing.T) { assert.Equal(t, autoscaling.NoRequeue, result) assert.NoError(t, err) } + +func TestStabilizeRecommendations(t *testing.T) { + currentTime := time.Now() + + tests := []struct { + name string + actions []datadoghq.DatadogPodAutoscalerHorizontalAction + currentReplicas int32 + recReplicas int32 + expected int32 + expectedReason string + upscaleWindow int32 + downscaleWindow int32 + scaleDirection scaleDirection + }{ + { + name: "no downscale stabilization - constant upscale", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-60*time.Second), 4, 5, 6), + newHorizontalAction(currentTime.Add(-30*time.Second), 6, 4, 4), + }, + currentReplicas: 4, + recReplicas: 8, + expected: 8, + expectedReason: "", + upscaleWindow: 0, + downscaleWindow: 300, + scaleDirection: scaleUp, + }, + { + name: "downscale stabilization", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-60*time.Second), 8, 6, 6), + newHorizontalAction(currentTime.Add(-30*time.Second), 6, 5, 5), + }, + currentReplicas: 5, + recReplicas: 4, + expected: 5, + expectedReason: "desired replica count limited to 5 (originally 4) due to stabilization window", + upscaleWindow: 0, + downscaleWindow: 300, + scaleDirection: scaleDown, + }, + { + name: "downscale stabilization, recommendation flapping", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-90*time.Second), 8, 6, 6), + newHorizontalAction(currentTime.Add(-60*time.Second), 6, 9, 9), + newHorizontalAction(currentTime.Add(-30*time.Second), 9, 7, 7), + }, + currentReplicas: 7, + recReplicas: 5, + expected: 7, + expectedReason: "desired replica count limited to 7 (originally 5) due to stabilization window", + upscaleWindow: 0, + downscaleWindow: 300, + scaleDirection: scaleDown, + }, + { + name: "upscale stabilization", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-60*time.Second), 5, 6, 6), + newHorizontalAction(currentTime.Add(-30*time.Second), 6, 8, 8), + }, + currentReplicas: 8, + recReplicas: 12, + expected: 8, + expectedReason: "desired replica count limited to 8 (originally 12) due to stabilization window", + upscaleWindow: 300, + downscaleWindow: 0, + scaleDirection: scaleUp, + }, + { + name: "upscale stabilization, recommendation flapping", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-90*time.Second), 6, 8, 8), + newHorizontalAction(currentTime.Add(-60*time.Second), 8, 7, 7), + newHorizontalAction(currentTime.Add(-30*time.Second), 7, 9, 9), + }, + currentReplicas: 9, + recReplicas: 12, + expected: 9, + expectedReason: "desired replica count limited to 9 (originally 12) due to stabilization window", + upscaleWindow: 300, + downscaleWindow: 0, + scaleDirection: scaleUp, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + recommendedReplicas, limitReason := stabilizeRecommendations(currentTime, tt.actions, tt.currentReplicas, tt.recReplicas, tt.upscaleWindow, tt.downscaleWindow, tt.scaleDirection) + assert.Equal(t, tt.expected, recommendedReplicas) + assert.Equal(t, tt.expectedReason, limitReason) + }) + } +} + +func TestHorizontalControllerSyncDownscaleWithStabilization(t *testing.T) { + testTime := time.Now() + startTime := testTime.Add(-time.Hour) + defaultStepDuration := 30 * time.Second + + f := newHorizontalControllerFixture(t, startTime) + autoscalerNamespace := "default" + autoscalerName := "test" + + expectedGVK := schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + } + + fakePai := &model.FakePodAutoscalerInternal{ + Namespace: autoscalerNamespace, + Name: autoscalerName, + Spec: &datadoghq.DatadogPodAutoscalerSpec{ + TargetRef: v2.CrossVersionObjectReference{ + Name: autoscalerName, + Kind: expectedGVK.Kind, + APIVersion: expectedGVK.Group + "/" + expectedGVK.Version, + }, + Constraints: &datadoghq.DatadogPodAutoscalerConstraints{ + MinReplicas: pointer.Ptr[int32](90), + MaxReplicas: 120, + }, + Policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 300, + }, + }, + }, + HorizontalLastActions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(f.clock.Now().Add(-60*time.Second), 90, 94, 94), + newHorizontalAction(f.clock.Now().Add(-30*time.Second), 94, 97, 97), + }, + ScalingValues: model.ScalingValues{ + Horizontal: &model.HorizontalScalingValues{ + Source: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + Timestamp: f.clock.Now().Add(-defaultStepDuration), + Replicas: 100, + }, + }, + TargetGVK: expectedGVK, + HorizontalEventsRetention: 5 * time.Minute, + } + + // Test upscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err := f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 97, + statusReplicas: 97, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 97 replicas, limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 97, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 97) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 95 replicas, still limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 95, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 95) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 92 replicas (not limited) + // Moving clock 4 minutes forward to get past stabilization window + f.clock.Step(4 * time.Minute) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 92, + scaleReplicas: 92, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 92, + statusReplicas: 92, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) +} + +func TestHorizontalControllerSyncUpscaleWithStabilization(t *testing.T) { + testTime := time.Now() + startTime := testTime.Add(-time.Hour) + defaultStepDuration := 30 * time.Second + + f := newHorizontalControllerFixture(t, startTime) + autoscalerNamespace := "default" + autoscalerName := "test" + + expectedGVK := schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + } + + fakePai := &model.FakePodAutoscalerInternal{ + Namespace: autoscalerNamespace, + Name: autoscalerName, + Spec: &datadoghq.DatadogPodAutoscalerSpec{ + TargetRef: v2.CrossVersionObjectReference{ + Name: autoscalerName, + Kind: expectedGVK.Kind, + APIVersion: expectedGVK.Group + "/" + expectedGVK.Version, + }, + Constraints: &datadoghq.DatadogPodAutoscalerConstraints{ + MinReplicas: pointer.Ptr[int32](90), + MaxReplicas: 120, + }, + Policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 300, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + }, + }, + HorizontalLastActions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(f.clock.Now().Add(-60*time.Second), 120, 110, 110), + newHorizontalAction(f.clock.Now().Add(-30*time.Second), 110, 104, 104), + }, + ScalingValues: model.ScalingValues{ + Horizontal: &model.HorizontalScalingValues{ + Source: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + Timestamp: f.clock.Now().Add(-defaultStepDuration), + Replicas: 100, + }, + }, + TargetGVK: expectedGVK, + HorizontalEventsRetention: 5 * time.Minute, + } + + // Test downscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err := f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 104, + statusReplicas: 104, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 102 replicas, limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 102, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 102) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 105 replicas, still limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 105, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 105) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 102 replicas (not limited) + // Moving clock 4 minutes forward to get past stabilization window + f.clock.Step(4 * time.Minute) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 102, + scaleReplicas: 102, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 102, + statusReplicas: 102, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) +} diff --git a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go index a1dc2f722beb1..0b36ab17ae2ef 100644 --- a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go +++ b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go @@ -589,16 +589,14 @@ func getHorizontalEventsRetention(policy *datadoghq.DatadogPodAutoscalerPolicy, if policy.Upscale != nil { upscaleRetention := getLongestScalingRulesPeriod(policy.Upscale.Rules) - if upscaleRetention > longestRetention { - longestRetention = upscaleRetention - } + upscaleStabilizationWindow := time.Second * time.Duration(policy.Upscale.StabilizationWindowSeconds) + longestRetention = max(longestRetention, upscaleRetention, upscaleStabilizationWindow) } if policy.Downscale != nil { downscaleRetention := getLongestScalingRulesPeriod(policy.Downscale.Rules) - if downscaleRetention > longestRetention { - longestRetention = downscaleRetention - } + downscaleStabilizationWindow := time.Second * time.Duration(policy.Downscale.StabilizationWindowSeconds) + longestRetention = max(longestRetention, downscaleRetention, downscaleStabilizationWindow) } if longestRetention > longestLookbackAllowed { diff --git a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go index 15dc2a99bea8d..966f23b46c641 100644 --- a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go +++ b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go @@ -76,3 +76,193 @@ func TestAddHorizontalAction(t *testing.T) { *addedAction2, }, horizontalLastActions) } + +func TestGetHorizontalEventsRetention(t *testing.T) { + tests := []struct { + name string + policy *datadoghq.DatadogPodAutoscalerPolicy + longestLookbackAllowed time.Duration + expectedRetention time.Duration + }{ + { + name: "No policy, no retention", + policy: nil, + longestLookbackAllowed: 0, + expectedRetention: 0, + }, + { + name: "No policy, 15 minutes retention", + policy: nil, + longestLookbackAllowed: 15 * time.Minute, + expectedRetention: 0, + }, + { + name: "Upscale policy with rules, 30 minutes retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 15 * time.Minute, + }, + { + name: "Upscale policy with rules, 10 minutes max retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 10 * time.Minute, + expectedRetention: 10 * time.Minute, + }, + { + name: "Upscale and downscale policy with rules, 30 minutes retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 960, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 16 * time.Minute, + }, + { + name: "Upscale and downscale policy with rules, 10 minutes max retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 960, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 10 * time.Minute, + expectedRetention: 10 * time.Minute, + }, + { + name: "Upscale stabilization window 5 minutes", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + StabilizationWindowSeconds: 300, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 5 * time.Minute, + }, + { + name: "Downscale stabilization window 5 minutes", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + StabilizationWindowSeconds: 300, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 5 * time.Minute, + }, + { + name: "Stabilization, rules, max retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 360, + Value: 2, + }, + }, + StabilizationWindowSeconds: 300, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 420, + Value: 2, + }, + }, + StabilizationWindowSeconds: 180, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 7 * time.Minute, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + horizontalEventsRetention := getHorizontalEventsRetention(tt.policy, tt.longestLookbackAllowed) + assert.Equal(t, tt.expectedRetention, horizontalEventsRetention) + }) + } +} diff --git a/pkg/collector/corechecks/embed/apm/apm.go b/pkg/collector/corechecks/embed/apm/apm.go index 28162165eee6d..b724e77720b63 100644 --- a/pkg/collector/corechecks/embed/apm/apm.go +++ b/pkg/collector/corechecks/embed/apm/apm.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build apm && !windows && !linux +//go:build darwin // Package apm contains the APM check package apm diff --git a/pkg/collector/corechecks/embed/apm/apm_nix.go b/pkg/collector/corechecks/embed/apm/apm_nix.go index a024e45a11844..3dd1535ce8432 100644 --- a/pkg/collector/corechecks/embed/apm/apm_nix.go +++ b/pkg/collector/corechecks/embed/apm/apm_nix.go @@ -3,9 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build apm && !windows && !linux - -// linux handled by systemd/upstart +//go:build darwin package apm diff --git a/pkg/collector/corechecks/embed/apm/stub.go b/pkg/collector/corechecks/embed/apm/stub.go index b4c5ea584872b..5ed387c9a2d8f 100644 --- a/pkg/collector/corechecks/embed/apm/stub.go +++ b/pkg/collector/corechecks/embed/apm/stub.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !apm || windows || linux +//go:build !darwin // Package apm provides a stub for the APM check package apm diff --git a/pkg/collector/corechecks/embed/process/process_agent.go b/pkg/collector/corechecks/embed/process/process_agent.go index c27d5a4b8f975..0c4f971467a9e 100644 --- a/pkg/collector/corechecks/embed/process/process_agent.go +++ b/pkg/collector/corechecks/embed/process/process_agent.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build process && (darwin || freebsd) +//go:build darwin //nolint:revive // TODO(PLINT) Fix revive linter package process diff --git a/pkg/collector/corechecks/embed/process/stub.go b/pkg/collector/corechecks/embed/process/stub.go index 1faeaac1b12a6..2279f0ea3c370 100644 --- a/pkg/collector/corechecks/embed/process/stub.go +++ b/pkg/collector/corechecks/embed/process/stub.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !process || (!darwin && !freebsd) +//go:build !darwin //nolint:revive // TODO(Process) Fix revive linter package process diff --git a/pkg/collector/corechecks/net/network/network.go b/pkg/collector/corechecks/net/network/network.go index c3505185203aa..d4a1e5cd6ebe7 100644 --- a/pkg/collector/corechecks/net/network/network.go +++ b/pkg/collector/corechecks/net/network/network.go @@ -17,7 +17,7 @@ import ( "strconv" "strings" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" yaml "gopkg.in/yaml.v2" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" diff --git a/pkg/collector/corechecks/net/network/network_test.go b/pkg/collector/corechecks/net/network/network_test.go index b2476ca3d734c..4a033f97dbe86 100644 --- a/pkg/collector/corechecks/net/network/network_test.go +++ b/pkg/collector/corechecks/net/network/network_test.go @@ -10,7 +10,7 @@ package network import ( "testing" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/collector/corechecks/oracle/oracle.go b/pkg/collector/corechecks/oracle/oracle.go index a36830ceb39bb..224555ba85bd6 100644 --- a/pkg/collector/corechecks/oracle/oracle.go +++ b/pkg/collector/corechecks/oracle/oracle.go @@ -137,7 +137,7 @@ func handleServiceCheck(c *Check, err error) { status = servicecheck.ServiceCheckCritical log.Errorf("%s failed to connect: %s", c.logPrompt, err) } - sender.ServiceCheck("oracle.can_connect", status, "", c.tags, message) + sendServiceCheck(c, "oracle.can_connect", status, message) sender.Commit() } diff --git a/pkg/collector/corechecks/oracle/oracle_integration_test.go b/pkg/collector/corechecks/oracle/oracle_integration_test.go index c0df585ca2065..0d37c3d66ec50 100644 --- a/pkg/collector/corechecks/oracle/oracle_integration_test.go +++ b/pkg/collector/corechecks/oracle/oracle_integration_test.go @@ -424,6 +424,14 @@ func buildConnectionString(connectionConfig config.ConnectionConfig) string { return connStr } +func TestCanConnectTags(t *testing.T) { + chk, sender := newDefaultCheck(t, "", "") + err := chk.Run() + require.NoError(t, err) + + sender.AssertServiceCheck(t, "oracle.can_connect", servicecheck.ServiceCheckOK, chk.dbHostname, nil, "") +} + func TestLargeUint64Binding(t *testing.T) { var err error c, _ := newDefaultCheck(t, "", "") diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go index d40a5216f1355..357409b59712c 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go @@ -288,13 +288,23 @@ func TestPorts(t *testing.T) { startUDP("udp4") startUDP("udp6") + expectedPortsMap := make(map[uint16]struct{}, len(expectedPorts)) + serviceMap := getServicesMap(t, url) pid := os.Getpid() require.Contains(t, serviceMap, pid) for _, port := range expectedPorts { + expectedPortsMap[port] = struct{}{} assert.Contains(t, serviceMap[pid].Ports, port) } for _, port := range unexpectedPorts { + // An unexpected port number can also be expected since UDP and TCP and + // v4 and v6 are all in the same list. Just skip the extra check in that + // case since it should be rare. + if _, alsoExpected := expectedPortsMap[port]; alsoExpected { + continue + } + assert.NotContains(t, serviceMap[pid].Ports, port) } } diff --git a/pkg/collector/corechecks/snmp/integration_topology_test.go b/pkg/collector/corechecks/snmp/integration_topology_test.go index b4f0f42ceb6d0..c9298738ccf56 100644 --- a/pkg/collector/corechecks/snmp/integration_topology_test.go +++ b/pkg/collector/corechecks/snmp/integration_topology_test.go @@ -661,6 +661,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:101.1", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -689,6 +690,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:102.2", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -1363,6 +1365,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:1.5", "source_type": "cdp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -1386,6 +1389,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:2.3", "source_type": "cdp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -2056,6 +2060,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:101.1", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -2084,6 +2089,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:102.2", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" diff --git a/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go b/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go index 3e9ad39677d4c..6bdcec852a1d8 100644 --- a/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go +++ b/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go @@ -385,8 +385,9 @@ func buildNetworkTopologyMetadataWithLLDP(deviceID string, store *metadata.Store remEntryUniqueID := localPortNum + "." + lldpRemIndex newLink := devicemetadata.TopologyLinkMetadata{ - ID: deviceID + ":" + remEntryUniqueID, - SourceType: topologyLinkSourceTypeLLDP, + ID: deviceID + ":" + remEntryUniqueID, + SourceType: topologyLinkSourceTypeLLDP, + Integration: common.SnmpIntegrationName, Remote: &devicemetadata.TopologyLinkSide{ Device: &devicemetadata.TopologyLinkDevice{ Name: store.GetColumnAsString("lldp_remote.device_name", strIndex), @@ -446,8 +447,9 @@ func buildNetworkTopologyMetadataWithCDP(deviceID string, store *metadata.Store, remEntryUniqueID := cdpCacheIfIndex + "." + cdpCacheDeviceIndex newLink := devicemetadata.TopologyLinkMetadata{ - ID: deviceID + ":" + remEntryUniqueID, - SourceType: topologyLinkSourceTypeCDP, + ID: deviceID + ":" + remEntryUniqueID, + SourceType: topologyLinkSourceTypeCDP, + Integration: common.SnmpIntegrationName, Remote: &devicemetadata.TopologyLinkSide{ Device: &devicemetadata.TopologyLinkDevice{ Name: store.GetColumnAsString("cdp_remote.device_name", strIndex), diff --git a/pkg/collector/corechecks/system/cpu/cpu/context_switches.go b/pkg/collector/corechecks/system/cpu/cpu/context_switches.go new file mode 100644 index 0000000000000..ca83b826dd331 --- /dev/null +++ b/pkg/collector/corechecks/system/cpu/cpu/context_switches.go @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. +//go:build !linux && !windows + +package cpu + +import "fmt" + +// GetContextSwitches retrieves the number of context switches for the current process. +// It returns an integer representing the count and an error if the retrieval fails. +func GetContextSwitches() (int64, error) { + return 0, fmt.Errorf("context switches not supported on macOS") +} diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches_linux.go b/pkg/collector/corechecks/system/cpu/cpu/context_switches_linux.go similarity index 64% rename from pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches_linux.go rename to pkg/collector/corechecks/system/cpu/cpu/context_switches_linux.go index 97dd5eab205e5..b1442821092c4 100644 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches_linux.go +++ b/pkg/collector/corechecks/system/cpu/cpu/context_switches_linux.go @@ -9,17 +9,23 @@ package cpu import ( "bufio" "fmt" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/log" "os" - "path/filepath" "strconv" "strings" - - "github.com/DataDog/datadog-agent/pkg/aggregator/sender" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" ) -func readCtxSwitches(procStatPath string) (ctxSwitches int64, err error) { - file, err := os.Open(procStatPath) +// GetContextSwitches retrieves the number of context switches for the current process. +// It returns an integer representing the count and an error if the retrieval fails. +func GetContextSwitches() (ctxSwitches int64, err error) { + log.Debug("collecting ctx switches") + procfsPath := "/proc" + if pkgconfigsetup.Datadog().IsSet("procfs_path") { + procfsPath = pkgconfigsetup.Datadog().GetString("procfs_path") + } + filePath := procfsPath + "/stat" + file, err := os.Open(filePath) if err != nil { return 0, err } @@ -32,24 +38,10 @@ func readCtxSwitches(procStatPath string) (ctxSwitches int64, err error) { elemts := strings.Split(txt, " ") ctxSwitches, err = strconv.ParseInt(elemts[1], 10, 64) if err != nil { - return 0, fmt.Errorf("%s in '%s' at line %d", err, procStatPath, i) + return 0, fmt.Errorf("%s in '%s' at line %d", err, filePath, i) } return ctxSwitches, nil } } - return 0, fmt.Errorf("could not find the context switches in stat file") } - -func collectCtxSwitches(sender sender.Sender) error { - procfsPath := "/proc" - if pkgconfigsetup.Datadog().IsSet("procfs_path") { - procfsPath = pkgconfigsetup.Datadog().GetString("procfs_path") - } - ctxSwitches, err := readCtxSwitches(filepath.Join(procfsPath, "/stat")) - if err != nil { - return err - } - sender.MonotonicCount("system.cpu.context_switches", float64(ctxSwitches), "", nil) - return nil -} diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu.go b/pkg/collector/corechecks/system/cpu/cpu/cpu.go index 8ca44f462bd61..827f032bba9e5 100644 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu.go +++ b/pkg/collector/corechecks/system/cpu/cpu/cpu.go @@ -10,7 +10,8 @@ package cpu import ( "fmt" - "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v4/cpu" + "gopkg.in/yaml.v2" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" @@ -23,15 +24,20 @@ import ( const CheckName = "cpu" // For testing purpose -var cpuTimesFunc = cpu.Times -var cpuInfoFunc = cpu.Info +var getCpuTimes = cpu.Times +var getCpuInfo = cpu.Info +var getContextSwitches = GetContextSwitches // Check doesn't need additional fields type Check struct { core.CheckBase - nbCPU float64 - lastNbCycle float64 - lastTimes cpu.TimesStat + instanceConfig cpuInstanceConfig + lastNbCycle float64 + lastTimes cpu.TimesStat +} + +type cpuInstanceConfig struct { + ReportTotalPerCPU bool `yaml:"report_total_percpu"` } // Run executes the check @@ -40,40 +46,81 @@ func (c *Check) Run() error { if err != nil { return err } + c.reportContextSwitches(sender) + numCores, err := c.reportCpuInfo(sender) + if err != nil { + return err + } + err = c.reportCpuMetricsPercent(sender, numCores) + if err != nil { + return err + } + err = c.reportCpuMetricsTotal(sender) + if err != nil { + return err + } + sender.Commit() + return nil +} - err = collectCtxSwitches(sender) +func (c *Check) reportContextSwitches(sender sender.Sender) { + ctxSwitches, err := getContextSwitches() if err != nil { - log.Debugf("cpu.Check could not read context switches: %s", err.Error()) - // Don't return here, we still want to collect the CPU metrics even if we could not + log.Debugf("could not read context switches: %s", err.Error()) + // Don't return error here, we still want to collect the CPU metrics even if we could not // read the context switches + } else { + log.Debugf("getContextSwitches: %d", ctxSwitches) + sender.MonotonicCount("system.cpu.context_switches", float64(ctxSwitches), "", nil) + } +} + +func (c *Check) reportCpuInfo(sender sender.Sender) (numCores int32, err error) { + cpuInfo, err := getCpuInfo() + if err != nil { + log.Errorf("could not retrieve cpu info: %s", err.Error()) + return 0, err + } + log.Debugf("getCpuInfo: %s", cpuInfo) + numCores = 0 + for _, i := range cpuInfo { + numCores += i.Cores } + sender.Gauge("system.cpu.num_cores", float64(numCores), "", nil) + return numCores, nil +} - cpuTimes, err := cpuTimesFunc(false) +func (c *Check) reportCpuMetricsPercent(sender sender.Sender, numCores int32) (err error) { + cpuTimes, err := getCpuTimes(false) if err != nil { - log.Errorf("cpu.Check: could not retrieve cpu stats: %s", err) + log.Errorf("could not retrieve cpu times: %s", err.Error()) + return err + } + log.Debugf("getCpuTimes(false): %s", cpuTimes) + if len(cpuTimes) == 0 { + err = fmt.Errorf("no cpu stats retrieve (empty results)") + log.Errorf("%s", err.Error()) return err - } else if len(cpuTimes) < 1 { - errEmpty := fmt.Errorf("no cpu stats retrieve (empty results)") - log.Errorf("cpu.Check: %s", errEmpty) - return errEmpty } t := cpuTimes[0] - total := t.User + t.System + t.Idle + t.Nice + t.Iowait + t.Irq + t.Softirq + t.Steal - nbCycle := total / c.nbCPU - + log.Debugf("total: %f", total) + nbCycle := total / float64(numCores) + log.Debugf("nbCycle: %f", nbCycle) if c.lastNbCycle != 0 { // gopsutil return the sum of every CPU + log.Debugf("c.lastNbCycle: %f", c.lastNbCycle) toPercent := 100 / (nbCycle - c.lastNbCycle) + log.Debugf("toPercent: %f", toPercent) - user := ((t.User + t.Nice) - (c.lastTimes.User + c.lastTimes.Nice)) / c.nbCPU - system := ((t.System + t.Irq + t.Softirq) - (c.lastTimes.System + c.lastTimes.Irq + c.lastTimes.Softirq)) / c.nbCPU - interrupt := ((t.Irq + t.Softirq) - (c.lastTimes.Irq + c.lastTimes.Softirq)) / c.nbCPU - iowait := (t.Iowait - c.lastTimes.Iowait) / c.nbCPU - idle := (t.Idle - c.lastTimes.Idle) / c.nbCPU - stolen := (t.Steal - c.lastTimes.Steal) / c.nbCPU - guest := (t.Guest - c.lastTimes.Guest) / c.nbCPU + user := ((t.User + t.Nice) - (c.lastTimes.User + c.lastTimes.Nice)) / float64(numCores) + system := ((t.System + t.Irq + t.Softirq) - (c.lastTimes.System + c.lastTimes.Irq + c.lastTimes.Softirq)) / float64(numCores) + interrupt := ((t.Irq + t.Softirq) - (c.lastTimes.Irq + c.lastTimes.Softirq)) / float64(numCores) + iowait := (t.Iowait - c.lastTimes.Iowait) / float64(numCores) + idle := (t.Idle - c.lastTimes.Idle) / float64(numCores) + stolen := (t.Steal - c.lastTimes.Steal) / float64(numCores) + guest := (t.Guest - c.lastTimes.Guest) / float64(numCores) sender.Gauge("system.cpu.user", user*toPercent, "", nil) sender.Gauge("system.cpu.system", system*toPercent, "", nil) @@ -83,32 +130,43 @@ func (c *Check) Run() error { sender.Gauge("system.cpu.stolen", stolen*toPercent, "", nil) sender.Gauge("system.cpu.guest", guest*toPercent, "", nil) } - - sender.Gauge("system.cpu.num_cores", c.nbCPU, "", nil) - sender.Commit() - c.lastNbCycle = nbCycle c.lastTimes = t return nil } -// Configure the CPU check -func (c *Check) Configure(senderManager sender.SenderManager, _ uint64, data integration.Data, initConfig integration.Data, source string) error { - err := c.CommonConfigure(senderManager, initConfig, data, source) +func (c *Check) reportCpuMetricsTotal(sender sender.Sender) (err error) { + cpuTimes, err := getCpuTimes(c.instanceConfig.ReportTotalPerCPU) if err != nil { + log.Errorf("could not retrieve cpu times: %s", err.Error()) return err } - // NOTE: This runs before the python checks, so we should be good, but cpuInfo() - // on windows initializes COM to the multithreaded model. Therefore, - // if a python check has run on this native windows thread prior and - // CoInitialized() the thread to a different model (ie. single-threaded) - // This will cause cpuInfo() to fail. - info, err := cpuInfoFunc() + log.Debugf("getCpuTimes(%t): %s", c.instanceConfig.ReportTotalPerCPU, cpuTimes) + for _, t := range cpuTimes { + tags := []string{fmt.Sprintf("core:%s", t.CPU)} + sender.Gauge("system.cpu.user.total", t.User, "", tags) + sender.Gauge("system.cpu.nice.total", t.Nice, "", tags) + sender.Gauge("system.cpu.system.total", t.System, "", tags) + sender.Gauge("system.cpu.idle.total", t.Idle, "", tags) + sender.Gauge("system.cpu.iowait.total", t.Iowait, "", tags) + sender.Gauge("system.cpu.irq.total", t.Irq, "", tags) + sender.Gauge("system.cpu.softirq.total", t.Softirq, "", tags) + sender.Gauge("system.cpu.steal.total", t.Steal, "", tags) + sender.Gauge("system.cpu.guest.total", t.Guest, "", tags) + sender.Gauge("system.cpu.guestnice.total", t.GuestNice, "", tags) + } + return nil +} + +// Configure configures the network checks +func (c *Check) Configure(senderManager sender.SenderManager, _ uint64, rawInstance integration.Data, rawInitConfig integration.Data, source string) error { + err := c.CommonConfigure(senderManager, rawInitConfig, rawInstance, source) if err != nil { - return fmt.Errorf("cpu.Check: could not query CPU info") + return err } - for _, i := range info { - c.nbCPU += float64(i.Cores) + err = yaml.Unmarshal(rawInstance, &c.instanceConfig) + if err != nil { + return err } return nil } diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches.go b/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches.go deleted file mode 100644 index 21a57e5ac4cde..0000000000000 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches.go +++ /dev/null @@ -1,14 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. -//go:build !linux && !windows - -package cpu - -import "github.com/DataDog/datadog-agent/pkg/aggregator/sender" - -func collectCtxSwitches(_ sender.Sender) error { - // On non-linux systems, do nothing - return nil -} diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go b/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go index d5124637d6c4a..c656765354022 100644 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go +++ b/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go @@ -8,52 +8,70 @@ package cpu import ( "errors" - "os" - "path/filepath" - "runtime" "testing" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" "github.com/DataDog/datadog-agent/pkg/collector/check" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" - "github.com/DataDog/datadog-agent/pkg/metrics" - - "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v4/cpu" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) var ( - firstSample = []cpu.TimesStat{ + firstTotalSample = cpu.TimesStat{ + CPU: "cpu-total", + User: 29386, + Nice: 623, + System: 63584, + Idle: 96761, + Iowait: 12113, + Irq: 10, + Softirq: 1151, + Steal: 0, + Guest: 0, + GuestNice: 0, + } + secondTotalSample = cpu.TimesStat{ + CPU: "cpu-total", + User: 29586, + Nice: 625, + System: 68584, + Idle: 96761, + Iowait: 12153, + Irq: 15, + Softirq: 1451, + Steal: 2, + Guest: 0, + GuestNice: 0, + } + perCPUSamples = []cpu.TimesStat{ { - CPU: "cpu-total", - User: 1229386, - Nice: 623, - System: 263584, - Idle: 25496761, - Iowait: 12113, - Irq: 10, - Softirq: 1151, - Steal: 0, - Guest: 0, - GuestNice: 0, + CPU: "cpu0", + User: 83970.9, + Nice: 0.0, + System: 64060.9, + Idle: 208877.4, + Iowait: 12.1, + Irq: 43.5, + Softirq: 8.6, + Steal: 65.9, + Guest: 2.4, + GuestNice: 5.1, }, - } - secondSample = []cpu.TimesStat{ { - CPU: "cpu-total", - User: 1229586, - Nice: 625, - System: 268584, - Idle: 25596761, - Iowait: 12153, - Irq: 15, - Softirq: 1451, - Steal: 2, - Guest: 0, - GuestNice: 0, + CPU: "cpu1", + User: 82638.9, + Nice: 50.0, + System: 61564.1, + Idle: 212758.8, + Iowait: 1.2, + Irq: 2.3, + Softirq: 3.4, + Steal: 4.5, + Guest: 5.6, + GuestNice: 6.7, }, } cpuInfo = []cpu.InfoStat{ @@ -81,207 +99,328 @@ func createCheck() check.Check { return cpuCheck } -func TestCPUCheckLinuxFirstRunOk(t *testing.T) { - cpuInfoFunc = func() ([]cpu.InfoStat, error) { +func setupDefaultMocks() { + getContextSwitches = func() (int64, error) { + return 4, nil + } + getCpuInfo = func() ([]cpu.InfoStat, error) { return cpuInfo, nil } + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil + } +} + +func TestCPUCheckLinuxErrorInInstanceConfig(t *testing.T) { + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + + err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`min_collection_interval: "string_value"`), nil, "test") + + assert.NotNil(t, err) +} + +func TestCPUCheckLinuxErrorReportTotalPerCPUConfigNotBoolean(t *testing.T) { cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - if runtime.GOOS == "linux" { - m.On(metrics.MonotonicCountType.String(), "system.cpu.context_switches", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) + + err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: "string_value"`), nil, "test") + + assert.NotNil(t, err) +} + +func TestCPUCheckLinuxErrorStoppedSender(t *testing.T) { + stoppedSenderError := errors.New("demultiplexer is stopped") + getCpuInfo = func() ([]cpu.InfoStat, error) { + return cpuInfo, nil } - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil - } + m.GetSenderManager().(*aggregator.AgentDemultiplexer).Stop(false) err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) + assert.Equal(t, stoppedSenderError, err) } -func TestCPUCheckLinuxTwoRunsOk(t *testing.T) { - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil +func TestContextSwitchesError(t *testing.T) { + setupDefaultMocks() + getContextSwitches = func() (int64, error) { + return 0, errors.New("GetContextSwitches error") } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - if runtime.GOOS == "linux" { - m.On(metrics.MonotonicCountType.String(), "system.cpu.context_switches", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(2) - } - m.On(metrics.GaugeType.String(), "system.cpu.user", 0.1913803067769472, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.system", 5.026101621048045, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.interrupt", 0.2889653146879648, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.iowait", 0.03789709045088063, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.idle", 94.74272612720159, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.stolen", 0.0018948545225440318, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.guest", 0.0, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", 1.0, "", []string(nil)).Return().Times(2) - m.On("Commit").Return().Times(2) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil - } - cpuCheck.Run() - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return secondSample, nil - } err := cpuCheck.Run() assert.Nil(t, err) - m.AssertExpectations(t) + m.AssertNotCalled(t, "MonotonicCount", "system.cpu.context_switches", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorInInstanceConfig(t *testing.T) { +func TestContextSwitchesOk(t *testing.T) { + setupDefaultMocks() cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() - err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`min_collection_interval: "string_value"`), nil, "test") + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + err := cpuCheck.Run() - assert.NotNil(t, err) + assert.Nil(t, err) + m.AssertMetric(t, "MonotonicCount", "system.cpu.context_switches", 4, "", []string(nil)) } -func TestCPUCheckLinuxErrorInCpuInfo(t *testing.T) { +func TestNumCoresError(t *testing.T) { + setupDefaultMocks() cpuInfoError := errors.New("cpu.Check: could not query CPU info") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { + getCpuInfo = func() ([]cpu.InfoStat, error) { return nil, cpuInfoError } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() - err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + err := cpuCheck.Run() assert.Equal(t, cpuInfoError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.num_cores", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorStoppedSender(t *testing.T) { - stoppedSenderError := errors.New("demultiplexer is stopped") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } +func TestNumCoresOk(t *testing.T) { + setupDefaultMocks() cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - m.GetSenderManager().(*aggregator.AgentDemultiplexer).Stop(false) err := cpuCheck.Run() - assert.Equal(t, stoppedSenderError, err) + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.num_cores", 1, "", nil) } -func TestCPUCheckLinuxErrorProcFsPathNoExists(t *testing.T) { - pkgconfigsetup.Datadog().SetDefault("procfs_path", "/tmp") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil +func TestSystemCpuMetricsError(t *testing.T) { + setupDefaultMocks() + cpuTimesError := errors.New("cpu.Check: could not query CPU times") + getCpuTimes = func(bool) ([]cpu.TimesStat, error) { + return nil, cpuTimesError } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) - m.AssertNumberOfCalls(t, metrics.MonotonicCountType.String(), 0) + assert.Equal(t, cpuTimesError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.interrupt", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.stolen", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorProcFsPathEmptyFile(t *testing.T) { - tempFilePath := filepath.Join(os.TempDir(), "stat") - tempFile, err := os.Create(tempFilePath) - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(tempFile.Name()) - pkgconfigsetup.Datadog().SetDefault("procfs_path", os.TempDir()) - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil +func TestSystemCpuMetricsEmpty(t *testing.T) { + setupDefaultMocks() + expectedError := errors.New("no cpu stats retrieve (empty results)") + getCpuTimes = func(bool) ([]cpu.TimesStat, error) { + return []cpu.TimesStat{}, nil } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - err = cpuCheck.Run() + err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) - m.AssertNumberOfCalls(t, metrics.MonotonicCountType.String(), 0) + assert.Equal(t, expectedError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.interrupt", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.stolen", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorProcFsPathWrongFormat(t *testing.T) { - tempFilePath := filepath.Join(os.TempDir(), "stat") - tempFile, err := os.Create(tempFilePath) - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(tempFile.Name()) - _, err = tempFile.WriteString("ctxt string_value\n") - if err != nil { - t.Fatal("Error writing to temporary file:", err) - } - pkgconfigsetup.Datadog().SetDefault("procfs_path", os.TempDir()) - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil - } +func TestSystemCpuMetricsNotReportedOnFirstCheck(t *testing.T) { + setupDefaultMocks() cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - err = cpuCheck.Run() + err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) - m.AssertNumberOfCalls(t, metrics.MonotonicCountType.String(), 0) + assert.Equal(t, nil, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.interrupt", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.stolen", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorInCpuTimes(t *testing.T) { - cpuTimesError := errors.New("cpu Times error") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return nil, cpuTimesError +func TestSystemCpuMetricsReportedOnSecondCheck(t *testing.T) { + setupDefaultMocks() + firstCall := true + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + if firstCall { + firstCall = false + return []cpu.TimesStat{firstTotalSample}, nil + } + return []cpu.TimesStat{secondTotalSample}, nil } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + cpuCheck.Run() err := cpuCheck.Run() - assert.Equal(t, cpuTimesError, err) + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user", 3.640295548747522, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.system", 95.60281131735448, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.interrupt", 5.496485853306902, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.iowait", 0.7208506037123806, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.idle", 0.0, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.stolen", 0.03604253018561903, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.guest", 0.0, "", []string(nil)) } -func TestCPUCheckLinuxEmptyCpuTimes(t *testing.T) { - expectedError := errors.New("no cpu stats retrieve (empty results)") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil +func TestSystemCpuMetricsPerCpuError(t *testing.T) { + setupDefaultMocks() + cpuTimesError := errors.New("cpu.Check: could not query CPU times") + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return nil, cpuTimesError + } + return []cpu.TimesStat{firstTotalSample}, nil } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return []cpu.TimesStat{}, nil + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() + + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: true`), nil, "test") + err := cpuCheck.Run() + + assert.Equal(t, cpuTimesError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.nice.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.irq.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.softirq.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.steal.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guestnice.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + +} + +func TestSystemCpuMetricsPerCpuDefault(t *testing.T) { + setupDefaultMocks() + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") err := cpuCheck.Run() - assert.Equal(t, expectedError, err) + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 29386, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 623, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 63584, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 96761, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 12113, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 10, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 1151, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 0.0, "", []string{"core:cpu-total"}) +} +func TestSystemCpuMetricsPerCpuFalse(t *testing.T) { + setupDefaultMocks() + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil + } + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() + + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: false`), nil, "test") + err := cpuCheck.Run() + + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 29386, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 623, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 63584, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 96761, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 12113, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 10, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 1151, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 0.0, "", []string{"core:cpu-total"}) +} +func TestSystemCpuMetricsPerCpuTrue(t *testing.T) { + setupDefaultMocks() + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil + } + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() + + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: true`), nil, "test") + err := cpuCheck.Run() + + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 83970.9, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 82638.9, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 0.0, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 50.0, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 64060.9, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 61564.1, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 208877.4, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 212758.8, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 12.1, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 1.2, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 43.5, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 2.3, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 8.6, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 3.4, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 65.9, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 4.5, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 2.4, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 5.6, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 5.1, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 6.7, "", []string{"core:cpu1"}) } diff --git a/pkg/collector/corechecks/system/cpu/load/load.go b/pkg/collector/corechecks/system/cpu/load/load.go index 6d6aa1833dedb..52be3ea30e08a 100644 --- a/pkg/collector/corechecks/system/cpu/load/load.go +++ b/pkg/collector/corechecks/system/cpu/load/load.go @@ -10,8 +10,8 @@ package load import ( "fmt" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/load" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" diff --git a/pkg/collector/corechecks/system/cpu/load/load_test.go b/pkg/collector/corechecks/system/cpu/load/load_test.go index af820e8fc21fa..fd140edd86ffd 100644 --- a/pkg/collector/corechecks/system/cpu/load/load_test.go +++ b/pkg/collector/corechecks/system/cpu/load/load_test.go @@ -9,9 +9,9 @@ package load import ( "testing" - "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v4/load" - "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v4/cpu" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" diff --git a/pkg/collector/corechecks/system/disk/disk/disk_nix.go b/pkg/collector/corechecks/system/disk/disk/disk_nix.go index 23a03601724a1..7a384f377056c 100644 --- a/pkg/collector/corechecks/system/disk/disk/disk_nix.go +++ b/pkg/collector/corechecks/system/disk/disk/disk_nix.go @@ -11,7 +11,7 @@ import ( "fmt" "path/filepath" - "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v4/disk" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" diff --git a/pkg/collector/corechecks/system/disk/disk/disk_test.go b/pkg/collector/corechecks/system/disk/disk/disk_test.go index 76aae7d0cfc36..2934b7fe9ed11 100644 --- a/pkg/collector/corechecks/system/disk/disk/disk_test.go +++ b/pkg/collector/corechecks/system/disk/disk/disk_test.go @@ -10,7 +10,7 @@ import ( "regexp" "testing" - "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v4/disk" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" diff --git a/pkg/collector/corechecks/system/disk/io/iostats.go b/pkg/collector/corechecks/system/disk/io/iostats.go index 2cdf7647bf14a..b40315ebff0af 100644 --- a/pkg/collector/corechecks/system/disk/io/iostats.go +++ b/pkg/collector/corechecks/system/disk/io/iostats.go @@ -20,7 +20,7 @@ import ( ) const ( - // SectorSize is used here to substitute non-exporeted from github.com/shirou/gopsutil/v3/disk package constant named "sectorSize" + // SectorSize is used here to substitute non-exporeted from github.com/shirou/gopsutil/v4/disk package constant named "sectorSize" SectorSize = 512 kB = (1 << 10) CheckName = "io" diff --git a/pkg/collector/corechecks/system/disk/io/iostats_nix.go b/pkg/collector/corechecks/system/disk/io/iostats_nix.go index 8fe5b1e66da5e..e3813f162da9e 100644 --- a/pkg/collector/corechecks/system/disk/io/iostats_nix.go +++ b/pkg/collector/corechecks/system/disk/io/iostats_nix.go @@ -13,8 +13,8 @@ import ( "regexp" "time" - "github.com/shirou/gopsutil/v3/disk" - "github.com/shirou/gopsutil/v3/mem" // for system.io.block_{in,out} + "github.com/shirou/gopsutil/v4/disk" + "github.com/shirou/gopsutil/v4/mem" // for system.io.block_{in,out} "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" diff --git a/pkg/collector/corechecks/system/disk/io/iostats_nix_test.go b/pkg/collector/corechecks/system/disk/io/iostats_nix_test.go index 21bfbef4109da..61a60ccab654d 100644 --- a/pkg/collector/corechecks/system/disk/io/iostats_nix_test.go +++ b/pkg/collector/corechecks/system/disk/io/iostats_nix_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v4/disk" "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" diff --git a/pkg/collector/corechecks/system/disk/io/iostats_test.go b/pkg/collector/corechecks/system/disk/io/iostats_test.go index 005b4aad3ebda..bb7d3eaef18b7 100644 --- a/pkg/collector/corechecks/system/disk/io/iostats_test.go +++ b/pkg/collector/corechecks/system/disk/io/iostats_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - "github.com/shirou/gopsutil/v3/disk" - "github.com/shirou/gopsutil/v3/mem" + "github.com/shirou/gopsutil/v4/disk" + "github.com/shirou/gopsutil/v4/mem" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator" diff --git a/pkg/collector/corechecks/system/memory/memory_nix.go b/pkg/collector/corechecks/system/memory/memory_nix.go index 190a4c7e02b41..577fb222bb97b 100644 --- a/pkg/collector/corechecks/system/memory/memory_nix.go +++ b/pkg/collector/corechecks/system/memory/memory_nix.go @@ -11,7 +11,7 @@ import ( "fmt" "runtime" - "github.com/shirou/gopsutil/v3/mem" + "github.com/shirou/gopsutil/v4/mem" "github.com/DataDog/datadog-agent/pkg/util/log" diff --git a/pkg/collector/corechecks/system/memory/memory_nix_test.go b/pkg/collector/corechecks/system/memory/memory_nix_test.go index b7cfd2e55a3f9..e8be54dbb93a9 100644 --- a/pkg/collector/corechecks/system/memory/memory_nix_test.go +++ b/pkg/collector/corechecks/system/memory/memory_nix_test.go @@ -11,7 +11,7 @@ import ( "fmt" "testing" - "github.com/shirou/gopsutil/v3/mem" + "github.com/shirou/gopsutil/v4/mem" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/collector/corechecks/system/uptime/uptime_nix.go b/pkg/collector/corechecks/system/uptime/uptime_nix.go index cc121ac42de61..41c8421153e3b 100644 --- a/pkg/collector/corechecks/system/uptime/uptime_nix.go +++ b/pkg/collector/corechecks/system/uptime/uptime_nix.go @@ -7,7 +7,7 @@ package uptime import ( - "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v4/host" ) // For testing purpose diff --git a/pkg/collector/python/init_windows.go b/pkg/collector/python/init_windows.go index 04d18175df1f7..3de1ba7a459b0 100644 --- a/pkg/collector/python/init_windows.go +++ b/pkg/collector/python/init_windows.go @@ -9,8 +9,10 @@ package python import ( "os" + "path/filepath" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/winutil" ) // Any platform-specific initialization belongs here. @@ -21,5 +23,41 @@ func initializePlatform() error { os.Unsetenv("PYTHONPATH") } + // only use cache file when not admin + admin, _ := winutil.IsUserAnAdmin() + if !admin { + err := enableSeparatePythonCacheDir() + if err != nil { + return err + } + } + return nil } + +// enableSeparatePythonCacheDir configures Python to use a separate directory for its pycache. +// +// Creates a python-cache subdir in the configuration directory and configures Python to use it via the PYTHONPYCACHEPREFIX env var. +// +// https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPYCACHEPREFIX +func enableSeparatePythonCacheDir() error { + pd, err := winutil.GetProgramDataDir() + if err != nil { + return err + } + pycache := filepath.Join(pd, "python-cache") + + // check if path exists and create directory if it doesn't + if _, err := os.Stat(pycache); os.IsNotExist(err) { + if err := os.MkdirAll(pycache, 0755); err != nil { + return err + } + } else if err != nil { + return err + } + + os.Setenv("PYTHONPYCACHEPREFIX", pycache) + + return nil + +} diff --git a/pkg/config/autodiscovery/autodiscovery.go b/pkg/config/autodiscovery/autodiscovery.go index c1fab80232a55..e0feabf9968ff 100644 --- a/pkg/config/autodiscovery/autodiscovery.go +++ b/pkg/config/autodiscovery/autodiscovery.go @@ -9,6 +9,8 @@ package autodiscovery import ( + "errors" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" "github.com/DataDog/datadog-agent/pkg/config/env" @@ -77,7 +79,7 @@ func DiscoverComponentsFromConfig() ([]pkgconfigsetup.ConfigurationProviders, [] // Auto-activate autodiscovery without listeners: - snmp snmpConfig, err := snmplistener.NewListenerConfig() - if err != nil { + if err != nil && !errors.Is(err, snmplistener.ErrNoConfigGiven) { log.Errorf("Error unmarshalling snmp listener config. Error: %v", err) } else if len(snmpConfig.Configs) > 0 { detectedListeners = append(detectedListeners, pkgconfigsetup.Listeners{Name: "snmp"}) diff --git a/pkg/config/config_template.yaml b/pkg/config/config_template.yaml index a858e50ce8b7f..95d7f8c806e39 100644 --- a/pkg/config/config_template.yaml +++ b/pkg/config/config_template.yaml @@ -1024,13 +1024,13 @@ api_key: ## @env DD_LOGS_CONFIG_INTEGRATIONS_LOGS_FILES_MAX_SIZE - integer - optional - default: 10 ## The max size in MB that an integration logs file is allowed to use # - # integrations_logs_files_max_size + # integrations_logs_files_max_size: 10 ## @param integrations_logs_total_usage - integer - optional - default: 100 ## @env DD_LOGS_CONFIG_INTEGRATIONS_LOGS_TOTAL_USAGE - integer - optional - default: 100 ## The total combined usage all integrations logs files can use # - # integrations_logs_total_usage + # integrations_logs_total_usage: 100 {{ end -}} {{- if .TraceAgent }} @@ -1197,7 +1197,7 @@ api_key: # remove_query_string: false ## @param DD_APM_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS - boolean - optional ## If enabled, path segments in URLs containing digits are replaced by "?" - # remove_path_with_digits: false + # remove_paths_with_digits: false # # memcached: ## @param DD_APM_OBFUSCATION_MEMCACHED_ENABLED - boolean - optional @@ -1260,8 +1260,8 @@ api_key: ## The set of keys for which their values will be passed through SQL obfuscation # obfuscate_sql_values: # - val1 - # cache: - ## @param DD_APM_CACHE_ENABLED - boolean - optional + # cache: + ## @param DD_APM_OBFUSCATION_CACHE_ENABLED - boolean - optional ## Enables caching obfuscated statements. Currently supported for SQL and MongoDB queries. ## Enabled by default. # enabled: true @@ -1404,29 +1404,34 @@ api_key: # # port: 5012 - ## @param instrumentation_enabled - boolean - default: false - ## @env DD_APM_INSTRUMENTATION_ENABLED - boolean - default: false - ## Enables Single Step Instrumentation in the cluster (in beta) + ## @param instrumentation - custom object - optional + ## Specifies settings for Single Step Instrumentation. # - # instrumentation_enabled: false + # instrumentation: - ## @param instrumentation_enabled_namespaces - list of strings - optional - ## @env DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES - space separated list of strings - optional - ## Enables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is off in the whole cluster (in beta) - ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is false. Cannot be set together with DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES. - # - # instrumentation_enabled_namespaces: - # - ns1 - # - apps + ## @param enabled - boolean - default: false + ## @env DD_APM_INSTRUMENTATION_ENABLED - boolean - default: false + ## Enables Single Step Instrumentation in the cluster (in beta) + # + # enabled: false - ## @param instrumentation_disabled_namespaces - list of strings - optional - ## @env DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES - space separated list of strings - optional - ## Disables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is enabled in the whole cluster (in beta) - ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is true. Cannot be set together with DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES. - # - # instrumentation_disabled_namespaces: - # - ns2 - # - system-ns + ## @param enabled_namespaces - list of strings - optional + ## @env DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES - space separated list of strings - optional + ## Enables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is off in the whole cluster (in beta) + ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is false. Cannot be set together with DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES. + # + # enabled_namespaces: + # - ns1 + # - apps + + ## @param disabled_namespaces - list of strings - optional + ## @env DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES - space separated list of strings - optional + ## Disables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is enabled in the whole cluster (in beta) + ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is true. Cannot be set together with DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES. + # + # disabled_namespaces: + # - ns2 + # - system-ns ## @param trace_buffer - integer - optional - default: 0 ## @env DD_APM_TRACE_BUFFER - integer - optional - default: 0 @@ -1448,25 +1453,26 @@ api_key: ## Enables and configures the Probabilistic Sampler, compatible with the ## OTel Probabilistic Sampler Processor ( https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/probabilisticsamplerprocessor#probabilistic-sampling-processor ) ## - #probabilistic_sampler: - ## @env DD_APM_PROBABILISTIC_SAMPLER_ENABLED - boolean - optional - default: false - ## Enables or disables the probabilistic sampler - # enabled: false - # - ## @env DD_APM_PROBABILISTIC_SAMPLER_SAMPLING_PERCENTAGE - float - optional - default: 0 - ## Samples this percentage (0-100) of traffic - # sampling_percentage: 0 - # - ## @env DD_APM_PROBABILISTIC_SAMPLER_HASH_SEED - integer - optional - default: 0 - ## hash_seed: A seed used for the hash algorithm. This must match other agents and OTel - ## collectors using the probabilistic sampler to ensure consistent sampling. - # hash_seed: 0 + # probabilistic_sampler: + + ## @env DD_APM_PROBABILISTIC_SAMPLER_ENABLED - boolean - optional - default: false + ## Enables or disables the probabilistic sampler + # enabled: false + # + ## @env DD_APM_PROBABILISTIC_SAMPLER_SAMPLING_PERCENTAGE - float - optional - default: 0 + ## Samples this percentage (0-100) of traffic + # sampling_percentage: 0 + # + ## @env DD_APM_PROBABILISTIC_SAMPLER_HASH_SEED - integer - optional - default: 0 + ## hash_seed: A seed used for the hash algorithm. This must match other agents and OTel + ## collectors using the probabilistic sampler to ensure consistent sampling. + # hash_seed: 0 ## @param error_tracking_standalone - object - optional ## Enables Error Tracking Standalone ## - #error_tracking_standalone: - # + # error_tracking_standalone: + ## @param enabled - boolean - optional - default: false ## @env DD_APM_ERROR_TRACKING_STANDALONE_ENABLED - boolean - optional - default: false ## Enables or disables Error Tracking Standalone @@ -3613,7 +3619,7 @@ api_key: ## @param prometheus_scrape - custom object - optional ## This section configures the Autodiscovery based on the Prometheus annotations # -# prometheus: +# prometheus_scrape: ## @param enabled - boolean - optional - default: false ## Enables the prometheus config provider @@ -3864,50 +3870,50 @@ api_key: ## @param ignored_ip_addresses - list of strings - optional ## A list of IP addresses to ignore when scanning the network. # - # ignored_ip_addresses: - # - - # - + # ignored_ip_addresses: + # - + # - ## @param port - integer - optional - default: 161 ## The UDP port to use when connecting to SNMP devices. # - # port: 161 + # port: 161 ## @param snmp_version - integer - optional - default: ## Set the version of the SNMP protocol. Available options are: `1`, `2` or `3`. ## If unset, the Agent tries to guess the correct version based on other configuration ## parameters, for example: if `user` is set, the Agent uses SNMP v3. # - # snmp_version: + # snmp_version: ## @param timeout - integer - optional - default: 5 ## The number of seconds before timing out. # - # timeout: 5 + # timeout: 5 ## @param retries - integer - optional - default: 3 ## The number of retries before failure. # - # retries: 3 + # retries: 3 ## @param community_string - string - optional ## Required for SNMP v1 & v2. ## Enclose the community string with single quote like below (to avoid special characters being interpreted). ## Ex: 'public' # - # community_string: '' + # community_string: '' ## @param user - string - optional ## The username to connect to your SNMP devices. ## SNMPv3 only. # - # user: + # user: ## @param authKey - string - optional ## The passphrase to use with your Authentication type. ## SNMPv3 only. # - # authKey: + # authKey: ## @param authProtocol - string - optional ## The authentication protocol to use when connecting to your SNMP devices. @@ -3915,13 +3921,13 @@ api_key: ## Defaults to MD5 when `authentication_key` is specified. ## SNMPv3 only. # - # authProtocol: + # authProtocol: ## @param privKey - string - optional ## The passphrase to use with your privacy protocol. ## SNMPv3 only. # - # privKey: + # privKey: ## @param privProtocol - string - optional ## The privacy protocol to use when connecting to your SNMP devices. @@ -3929,35 +3935,35 @@ api_key: ## Defaults to DES when `privacy_key` is specified. ## SNMPv3 only. # - # privProtocol: + # privProtocol: ## @param context_name - string - optional ## The name of your context (optional SNMP v3-only parameter). # - # context_name: + # context_name: ## @param tags - list of strings - optional ## A list of tags to attach to every metric and service check of all devices discovered in the subnet. ## ## Learn more about tagging at https://docs.datadoghq.com/tagging # - # tags: - # - : - # - : + # tags: + # - : + # - : ## @param ad_identifier - string - optional - default: snmp ## A unique identifier to attach to devices from that subnetwork. ## When configuring the SNMP integration in snmp.d/auto_conf.yaml, ## specify the corresponding ad_identifier at the top of the file. # - # ad_identifier: snmp + # ad_identifier: snmp ## @param loader - string - optional - default: python ## Check loader to use. Available loaders: ## - core: will use corecheck SNMP integration ## - python: will use python SNMP integration # - # loader: core + # loader: core ## @param min_collection_interval - number - optional - default: 15 ## This changes the collection interval for the check instances created from @@ -3966,7 +3972,7 @@ api_key: ## For more information, see: ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval # - # min_collection_interval: 15 + # min_collection_interval: 15 ## @param use_device_id_as_hostname - boolean - optional - default: false ## Use `device:` (device_id is composed of `:`) as `hostname` @@ -3974,12 +3980,12 @@ api_key: ## `host:device:` as tag). ## This option is needed for custom tags. # - # use_device_id_as_hostname: true + # use_device_id_as_hostname: true ## @param oid_batch_size - integer - optional - default: 5 ## The number of OIDs handled by each batch. # - # oid_batch_size: 5 + # oid_batch_size: 5 ## @param interface_configs - map - optional ## This option is used to override interface inbound/outbound speed and add interface tags @@ -3994,27 +4000,27 @@ api_key: ## - "testTagKey:testTagValue" ## - "tagKey2:tagValue2" # - # interface_config: - # "10.0.0.1": - # - match_field: name - # match_value: eth0 - # in_speed: 50 - # out_speed: 25 - # - match_field: index - # match_value: '10' - # in_speed: 50 - # out_speed: 25 - # "10.0.0.2": - # - match_field: name - # match_value: eth3 - # in_speed: 50 - # out_speed: 25 - # "10.0.0.3": - # - match_field: name - # match_value: eth4 - # tags: - # - "monitored:true" - # - "customKey:customValue" + # interface_config: + # "10.0.0.1": + # - match_field: name + # match_value: eth0 + # in_speed: 50 + # out_speed: 25 + # - match_field: index + # match_value: '10' + # in_speed: 50 + # out_speed: 25 + # "10.0.0.2": + # - match_field: name + # match_value: eth3 + # in_speed: 50 + # out_speed: 25 + # "10.0.0.3": + # - match_field: name + # match_value: eth4 + # tags: + # - "monitored:true" + # - "customKey:customValue" ## @param ping - custom object - optional ## Configure ICMP pings for all hosts in SNMP autodiscovery @@ -4028,17 +4034,17 @@ api_key: ## of system-probe for elevated privileges. See ## system-probe.yaml.example for details. # - # ping: - # enabled: true # Disabled by default - # timeout: 3000 # Timeout in milliseconds - # count: 2 # Number of ping packets to send per check run - # interval: 10 # Time between sending pings (up to `count` packets) in milliseconds - # linux: # Linux-specific configuration - # use_raw_socket: true # Send pings in a privileged fashion using a raw socket. - # # This may be required if your system doesn't support - # # sending pings in an unprivileged fashion (using a UDP socket). - # # If `use_raw_socket` is set to true, you MUST also enable - # # system-probe which has elevated privileges. To enable it, see system-probe.yaml.example. + # ping: + # enabled: true # Disabled by default + # timeout: 3000 # Timeout in milliseconds + # count: 2 # Number of ping packets to send per check run + # interval: 10 # Time between sending pings (up to `count` packets) in milliseconds + # linux: # Linux-specific configuration + # use_raw_socket: true # Send pings in a privileged fashion using a raw socket. + # # This may be required if your system doesn't support + # # sending pings in an unprivileged fashion (using a UDP socket). + # # If `use_raw_socket` is set to true, you MUST also enable + # # system-probe which has elevated privileges. To enable it, see system-probe.yaml.example. ## @param snmp_traps - custom object - optional diff --git a/pkg/config/env/go.mod b/pkg/config/env/go.mod index 637c78a705df1..7b16228030c49 100644 --- a/pkg/config/env/go.mod +++ b/pkg/config/env/go.mod @@ -31,7 +31,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect diff --git a/pkg/config/env/go.sum b/pkg/config/env/go.sum index d7d4a9a3701e0..4a2c4e5bdc610 100644 --- a/pkg/config/env/go.sum +++ b/pkg/config/env/go.sum @@ -81,8 +81,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/pkg/config/fetcher/from_processes.go b/pkg/config/fetcher/from_processes.go index 5b8088f41d970..95c24e283fb46 100644 --- a/pkg/config/fetcher/from_processes.go +++ b/pkg/config/fetcher/from_processes.go @@ -71,7 +71,7 @@ func TraceAgentConfig(config config.Reader) (string, error) { c := util.GetClient(false) c.Timeout = config.GetDuration("server_timeout") * time.Second - ipcAddressWithPort := fmt.Sprintf("http://127.0.0.1:%d/config", port) + ipcAddressWithPort := fmt.Sprintf("https://127.0.0.1:%d/config", port) client := settingshttp.NewClient(c, ipcAddressWithPort, "trace-agent", settingshttp.NewHTTPClientOptions(util.CloseConnection)) return client.FullConfig() diff --git a/pkg/config/mock/go.mod b/pkg/config/mock/go.mod index 48f1f452fa096..f4edf2bbb8c8e 100644 --- a/pkg/config/mock/go.mod +++ b/pkg/config/mock/go.mod @@ -61,7 +61,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/config/mock/go.sum b/pkg/config/mock/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/config/mock/go.sum +++ b/pkg/config/mock/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 282d7f34ff86f..3b809d121d819 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -100,7 +100,7 @@ require ( github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.6 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect @@ -141,7 +141,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tinylib/msgp v1.2.4 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index 84cc570d9ad84..176ccf5fcd4b4 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -149,8 +149,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -378,8 +378,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= diff --git a/pkg/config/remote/service/service.go b/pkg/config/remote/service/service.go index a379ac5b387ec..84377477da44c 100644 --- a/pkg/config/remote/service/service.go +++ b/pkg/config/remote/service/service.go @@ -111,50 +111,21 @@ func (s *Service) getNewDirectorRoots(uptane uptaneClient, currentVersion uint64 return roots, nil } -func (s *Service) getTargetFiles(uptane uptaneClient, products []rdata.Product, cachedTargetFiles []*pbgo.TargetFileMeta) ([]*pbgo.File, error) { - productSet := make(map[rdata.Product]struct{}) - for _, product := range products { - productSet[product] = struct{}{} - } - targets, err := uptane.Targets() +func (s *Service) getTargetFiles(uptaneClient uptaneClient, targetFilePaths []string) ([]*pbgo.File, error) { + files, err := uptaneClient.TargetFiles(targetFilePaths) if err != nil { return nil, err } - cachedTargets := make(map[string]data.FileMeta) - for _, cachedTarget := range cachedTargetFiles { - hashes := make(data.Hashes) - for _, hash := range cachedTarget.Hashes { - h, err := hex.DecodeString(hash.Hash) - if err != nil { - return nil, err - } - hashes[hash.Algorithm] = h - } - cachedTargets[cachedTarget.Path] = data.FileMeta{ - Hashes: hashes, - Length: cachedTarget.Length, - } - } + var configFiles []*pbgo.File - for targetPath, targetMeta := range targets { - configPathMeta, err := rdata.ParseConfigPath(targetPath) - if err != nil { - return nil, err - } - if _, inClientProducts := productSet[rdata.Product(configPathMeta.Product)]; inClientProducts { - if notEqualErr := tufutil.FileMetaEqual(cachedTargets[targetPath], targetMeta.FileMeta); notEqualErr == nil { - continue - } - fileContents, err := uptane.TargetFile(targetPath) - if err != nil { - return nil, err - } - configFiles = append(configFiles, &pbgo.File{ - Path: targetPath, - Raw: fileContents, - }) - } + for path, contents := range files { + // Note: This unconditionally succeeds as long as we don't change bufferDestination earlier + configFiles = append(configFiles, &pbgo.File{ + Path: path, + Raw: contents, + }) } + return configFiles, nil } @@ -211,6 +182,7 @@ type uptaneClient interface { StoredOrgUUID() (string, error) Targets() (data.TargetFiles, error) TargetFile(path string) ([]byte, error) + TargetFiles(files []string) (map[string][]byte, error) TargetsMeta() ([]byte, error) TargetsCustom() ([]byte, error) TUFVersionState() (uptane.TUFVersions, error) @@ -828,36 +800,30 @@ func (s *CoreAgentService) ClientGetConfigs(_ context.Context, request *pbgo.Cli if err != nil { return nil, err } - targetsRaw, err := s.uptane.TargetsMeta() + + directorTargets, err := s.uptane.Targets() if err != nil { return nil, err } - targetFiles, err := s.getTargetFiles(s.uptane, rdata.StringListToProduct(request.Client.Products), request.CachedTargetFiles) + matchedClientConfigs, err := executeTracerPredicates(request.Client, directorTargets) if err != nil { return nil, err } - directorTargets, err := s.uptane.Targets() + neededFiles, err := filterNeededTargetFiles(matchedClientConfigs, request.CachedTargetFiles, directorTargets) if err != nil { return nil, err } - matchedClientConfigs, err := executeTracerPredicates(request.Client, directorTargets) + + targetFiles, err := s.getTargetFiles(s.uptane, neededFiles) if err != nil { return nil, err } - // filter files to only return the ones that predicates marked for this client - matchedConfigsMap := make(map[string]interface{}) - for _, configPointer := range matchedClientConfigs { - matchedConfigsMap[configPointer] = struct{}{} - } - filteredFiles := make([]*pbgo.File, 0, len(matchedClientConfigs)) - for _, targetFile := range targetFiles { - if _, ok := matchedConfigsMap[targetFile.Path]; ok { - filteredFiles = append(filteredFiles, targetFile) - } + targetsRaw, err := s.uptane.TargetsMeta() + if err != nil { + return nil, err } - canonicalTargets, err := enforceCanonicalJSON(targetsRaw) if err != nil { return nil, err @@ -866,11 +832,42 @@ func (s *CoreAgentService) ClientGetConfigs(_ context.Context, request *pbgo.Cli return &pbgo.ClientGetConfigsResponse{ Roots: roots, Targets: canonicalTargets, - TargetFiles: filteredFiles, + TargetFiles: targetFiles, ClientConfigs: matchedClientConfigs, }, nil } +func filterNeededTargetFiles(neededConfigs []string, cachedTargetFiles []*pbgo.TargetFileMeta, tufTargets data.TargetFiles) ([]string, error) { + // Build an O(1) lookup of cached target files + cachedTargetsMap := make(map[string]data.FileMeta) + for _, cachedTarget := range cachedTargetFiles { + hashes := make(data.Hashes) + for _, hash := range cachedTarget.Hashes { + h, err := hex.DecodeString(hash.Hash) + if err != nil { + return nil, err + } + hashes[hash.Algorithm] = h + } + cachedTargetsMap[cachedTarget.Path] = data.FileMeta{ + Hashes: hashes, + Length: cachedTarget.Length, + } + } + + // We don't need to pull the raw contents if the client already has the exact version of the file cached + filteredList := make([]string, 0, len(neededConfigs)) + for _, path := range neededConfigs { + if notEqualErr := tufutil.FileMetaEqual(cachedTargetsMap[path], tufTargets[path].FileMeta); notEqualErr == nil { + continue + } + + filteredList = append(filteredList, path) + } + + return filteredList, nil +} + // ConfigGetState returns the state of the configuration and the director repos in the local store func (s *CoreAgentService) ConfigGetState() (*pbgo.GetStateConfigResponse, error) { state, err := s.uptane.State() @@ -1117,29 +1114,14 @@ func (c *HTTPClient) getUpdate( if tufVersions.DirectorTargets == currentTargetsVersion { return nil, nil } - roots, err := c.getNewDirectorRoots(c.uptane, currentRootVersion, tufVersions.DirectorRoot) - if err != nil { - return nil, err - } - targetsRaw, err := c.uptane.TargetsMeta() - if err != nil { - return nil, err - } - targetFiles, err := c.getTargetFiles(c.uptane, rdata.StringListToProduct(products), cachedTargetFiles) - if err != nil { - return nil, err - } - - canonicalTargets, err := enforceCanonicalJSON(targetsRaw) - if err != nil { - return nil, err - } + // Filter out files that either: + // - don't correspond to the product list the client is requesting + // - have expired directorTargets, err := c.uptane.Targets() if err != nil { return nil, err } - productsMap := make(map[string]struct{}) for _, product := range products { productsMap[product] = struct{}{} @@ -1165,14 +1147,33 @@ func (c *HTTPClient) getUpdate( configs = append(configs, path) } + span.SetTag("configs.returned", configs) + span.SetTag("configs.expired", expiredConfigs) + // Gather the files and map-ify them for the state data structure + targetFiles, err := c.getTargetFiles(c.uptane, configs) + if err != nil { + return nil, err + } fileMap := make(map[string][]byte, len(targetFiles)) for _, f := range targetFiles { fileMap[f.Path] = f.Raw } - span.SetTag("configs.returned", configs) - span.SetTag("configs.expired", expiredConfigs) + // Gather some TUF metadata files we need to send down + roots, err := c.getNewDirectorRoots(c.uptane, currentRootVersion, tufVersions.DirectorRoot) + if err != nil { + return nil, err + } + targetsRaw, err := c.uptane.TargetsMeta() + if err != nil { + return nil, err + } + canonicalTargets, err := enforceCanonicalJSON(targetsRaw) + if err != nil { + return nil, err + } + return &state.Update{ TUFRoots: roots, TUFTargets: canonicalTargets, diff --git a/pkg/config/remote/service/service_test.go b/pkg/config/remote/service/service_test.go index 40d8302db3192..34175eea2feba 100644 --- a/pkg/config/remote/service/service_test.go +++ b/pkg/config/remote/service/service_test.go @@ -116,6 +116,11 @@ func (m *mockUptane) TargetFile(path string) ([]byte, error) { return args.Get(0).([]byte), args.Error(1) } +func (m *mockUptane) TargetFiles(files []string) (map[string][]byte, error) { + args := m.Called(files) + return args.Get(0).(map[string][]byte), args.Error(1) +} + func (m *mockUptane) TargetsMeta() ([]byte, error) { args := m.Called() return args.Get(0).([]byte), args.Error(1) @@ -486,8 +491,8 @@ func TestService(t *testing.T) { }, nil) uptaneClient.On("DirectorRoot", uint64(3)).Return(root3, nil) uptaneClient.On("DirectorRoot", uint64(4)).Return(root4, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return(fileAPM1, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return(fileAPM2, nil) + + uptaneClient.On("TargetFiles", mock.MatchedBy(listsEqual([]string{"datadog/2/APM_SAMPLING/id/1", "datadog/2/APM_SAMPLING/id/2"}))).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": fileAPM1, "datadog/2/APM_SAMPLING/id/2": fileAPM2}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -513,7 +518,7 @@ func TestService(t *testing.T) { configResponse, err := service.ClientGetConfigs(context.Background(), &pbgo.ClientGetConfigsRequest{Client: client}) assert.NoError(t, err) assert.ElementsMatch(t, [][]byte{canonicalRoot3, canonicalRoot4}, configResponse.Roots) - assert.ElementsMatch(t, []*pbgo.File{{Path: "datadog/2/APM_SAMPLING/id/1", Raw: fileAPM1}, {Path: "datadog/2/APM_SAMPLING/id/2", Raw: fileAPM2}}, configResponse.TargetFiles) + assert.ElementsMatch(t, []*pbgo.File{{Path: "datadog/2/APM_SAMPLING/id/1", Raw: fileAPM1}, {Path: "datadog/2/APM_SAMPLING/id/2", Raw: fileAPM2}}, configResponse.TargetFiles, nil) assert.Equal(t, canonicalTargets, configResponse.Targets) assert.ElementsMatch(t, configResponse.ClientConfigs, @@ -597,8 +602,7 @@ func TestServiceClientPredicates(t *testing.T) { DirectorRoot: 1, DirectorTargets: 5, }, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return([]byte(``), nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return([]byte(``), nil) + uptaneClient.On("TargetFiles", mock.MatchedBy(listsEqual([]string{"datadog/2/APM_SAMPLING/id/1", "datadog/2/APM_SAMPLING/id/2"}))).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": []byte(``), "datadog/2/APM_SAMPLING/id/2": []byte(``)}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -887,8 +891,7 @@ func TestConfigExpiration(t *testing.T) { DirectorRoot: 1, DirectorTargets: 5, }, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return([]byte(``), nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return([]byte(``), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/APM_SAMPLING/id/1"}).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": []byte(``), "datadog/2/APM_SAMPLING/id/2": []byte(``)}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -1190,7 +1193,7 @@ func TestHTTPClientRecentUpdate(t *testing.T) { }, nil, ) - uptaneClient.On("TargetFile", "datadog/2/TESTING1/id/1").Return([]byte(`testing_1`), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/TESTING1/id/1"}).Return(map[string][]byte{"datadog/2/TESTING1/id/1": []byte(`testing_1`)}, nil) client := setupCDNClient(t, uptaneClient) defer client.Close() @@ -1236,7 +1239,7 @@ func TestHTTPClientUpdateSuccess(t *testing.T) { }, nil, ) - uptaneClient.On("TargetFile", "datadog/2/TESTING1/id/1").Return([]byte(`testing_1`), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/TESTING1/id/1"}).Return(map[string][]byte{"datadog/2/TESTING1/id/1": []byte(`testing_1`)}, nil) updateErr := fmt.Errorf("uh oh") if tt.updateSucceeds { @@ -1261,3 +1264,24 @@ func TestHTTPClientUpdateSuccess(t *testing.T) { }) } } + +func listsEqual(mustMatch []string) func(candidate []string) bool { + return func(candidate []string) bool { + if len(candidate) != len(mustMatch) { + return false + } + + candidateSet := make(map[string]struct{}) + for _, item := range candidate { + candidateSet[item] = struct{}{} + } + + for _, item := range mustMatch { + if _, ok := candidateSet[item]; !ok { + return false + } + } + + return true + } +} diff --git a/pkg/config/remote/uptane/client.go b/pkg/config/remote/uptane/client.go index b4da1c675b296..fcd1d3834b478 100644 --- a/pkg/config/remote/uptane/client.go +++ b/pkg/config/remote/uptane/client.go @@ -342,6 +342,37 @@ func (c *Client) TargetFile(path string) ([]byte, error) { return c.unsafeTargetFile(path) } +// TargetFiles returns the content of various multiple target files if the repository is in a +// verified state. +func (c *Client) TargetFiles(targetFiles []string) (map[string][]byte, error) { + c.Lock() + defer c.Unlock() + + err := c.verify() + if err != nil { + return nil, err + } + + // Build the storage space + destinations := make(map[string]client.Destination) + for _, path := range targetFiles { + destinations[path] = &bufferDestination{} + } + + err = c.directorTUFClient.DownloadBatch(destinations) + if err != nil { + return nil, err + } + + // Build the return type + files := make(map[string][]byte) + for path, contents := range destinations { + files[path] = contents.(*bufferDestination).Bytes() + } + + return files, nil +} + // TargetsMeta returns the current raw targets.json meta of this uptane client func (c *Client) TargetsMeta() ([]byte, error) { c.Lock() diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 24f8d81503459..fdb1d427e9d15 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -79,7 +79,7 @@ const ( DefaultRuntimePoliciesDir = "/etc/datadog-agent/runtime-security.d" // DefaultCompressorKind is the default compressor. Options available are 'zlib' and 'zstd' - DefaultCompressorKind = "zlib" + DefaultCompressorKind = "zstd" // DefaultZstdCompressionLevel is the default compression level for `zstd`. // Compression level 1 provides the lowest compression ratio, but uses much less RSS especially diff --git a/pkg/config/setup/go.mod b/pkg/config/setup/go.mod index b7c9ff41de81b..efed57c318b83 100644 --- a/pkg/config/setup/go.mod +++ b/pkg/config/setup/go.mod @@ -73,7 +73,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/pkg/config/setup/go.sum b/pkg/config/setup/go.sum index 7f15a6d221890..eb1a58f09f4b8 100644 --- a/pkg/config/setup/go.sum +++ b/pkg/config/setup/go.sum @@ -91,8 +91,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index bdc97ee3d902a..26ea84571d050 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -371,13 +371,12 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.raw_classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_ring_buffer"), true) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_amd64"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_arm64"), false) + eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), true) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_amd64")) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_arm64")) eventMonitorBindEnv(cfg, join(evNS, "event_stream.buffer_size")) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "envs_with_value"), []string{"LD_PRELOAD", "LD_LIBRARY_PATH", "PATH", "HISTSIZE", "HISTFILESIZE", "GLIBC_TUNABLES"}) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "runtime_compilation.enabled"), false) - eventMonitorBindEnv(cfg, join(evNS, "runtime_compilation.compiled_constants_enabled")) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.enabled"), true) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.ingress.enabled"), false) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.raw_packet.enabled"), false) diff --git a/pkg/config/setup/system_probe_cws.go b/pkg/config/setup/system_probe_cws.go index d2cc6276907d3..c689e1fd68e28 100644 --- a/pkg/config/setup/system_probe_cws.go +++ b/pkg/config/setup/system_probe_cws.go @@ -57,6 +57,7 @@ func initCWSSystemProbeConfig(cfg pkgconfigmodel.Config) { cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.min_timeout", "10m") cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.max_dump_size", 1750) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_cgroups_count", 5) + cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.cgroup_managers", []string{"docker", "podman", "containerd", "cri-o"}) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_event_types", []string{"exec", "open", "dns", "imds"}) cfg.BindEnv("runtime_security_config.activity_dump.cgroup_dump_timeout") // deprecated in favor of dump_duration cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.dump_duration", "900s") diff --git a/pkg/config/teeconfig/go.mod b/pkg/config/teeconfig/go.mod index 2484fb9cd55e3..daec9f976da68 100644 --- a/pkg/config/teeconfig/go.mod +++ b/pkg/config/teeconfig/go.mod @@ -33,6 +33,7 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/config/utils/go.mod b/pkg/config/utils/go.mod index 5fc90b2da63fd..ef80f1fde725b 100644 --- a/pkg/config/utils/go.mod +++ b/pkg/config/utils/go.mod @@ -66,7 +66,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/config/utils/go.sum b/pkg/config/utils/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/config/utils/go.sum +++ b/pkg/config/utils/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/ebpf/maps/generic_map.go b/pkg/ebpf/maps/generic_map.go index dfaa5a5cd0ee2..5f77ba9c4e958 100644 --- a/pkg/ebpf/maps/generic_map.go +++ b/pkg/ebpf/maps/generic_map.go @@ -130,8 +130,17 @@ func Map[K any, V any](m *ebpf.Map) (*GenericMap[K, V], error) { return nil, err } + // See if we can perform binary.Read on the key type. If we can't we can't use the batch API + // for this map + var kval K + keySupportsBatchAPI := canBinaryReadKey[K]() + if !keySupportsBatchAPI { + log.Warnf("Key type %T does not support binary.Read, batch API will not be used for this map", kval) + } + return &GenericMap[K, V]{ - m: m, + m: m, + keySupportsBatchAPI: keySupportsBatchAPI, }, nil } diff --git a/pkg/ebpf/maps/generic_map_test.go b/pkg/ebpf/maps/generic_map_test.go index 3bc59f494f8b6..aa760871e4d15 100644 --- a/pkg/ebpf/maps/generic_map_test.go +++ b/pkg/ebpf/maps/generic_map_test.go @@ -572,3 +572,19 @@ func TestIterateWithPointerKey(t *testing.T) { require.NoError(t, it.Err()) require.Equal(t, expectedNumbers, actualNumbers) } + +func TestGenericHashMapCanUseBatchAPI(t *testing.T) { + hash, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.Hash, + KeySize: 4, + ValueSize: 4, + MaxEntries: 10, + }) + require.NoError(t, err) + t.Cleanup(func() { hash.Close() }) + + gm, err := Map[int32, int32](hash) + require.NoError(t, err) + + require.Equal(t, BatchAPISupported(), gm.CanUseBatchAPI()) +} diff --git a/pkg/ebpf/uprobes/testutil.go b/pkg/ebpf/uprobes/testutil.go index 93b109b499aa3..2d4e8b85d4b25 100644 --- a/pkg/ebpf/uprobes/testutil.go +++ b/pkg/ebpf/uprobes/testutil.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "sync" "testing" "time" @@ -132,6 +133,21 @@ type FakeProcFSEntry struct { Command string Exe string Maps string + Env map[string]string +} + +// getEnvironContents returns the formatted contents of the /proc//environ file for the entry. +func (f *FakeProcFSEntry) getEnvironContents() string { + if len(f.Env) == 0 { + return "" + } + + formattedEnvVars := make([]string, 0, len(f.Env)) + for k, v := range f.Env { + formattedEnvVars = append(formattedEnvVars, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(formattedEnvVars, "\x00") + "\x00" } // CreateFakeProcFS creates a fake /proc filesystem with the given entries, useful for testing attachment to processes. @@ -145,16 +161,13 @@ func CreateFakeProcFS(t *testing.T, entries []FakeProcFSEntry) string { createFile(t, filepath.Join(baseDir, "comm"), entry.Command) createFile(t, filepath.Join(baseDir, "maps"), entry.Maps) createSymlink(t, entry.Exe, filepath.Join(baseDir, "exe")) + createFile(t, filepath.Join(baseDir, "environ"), entry.getEnvironContents()) } return procRoot } func createFile(t *testing.T, path, data string) { - if data == "" { - return - } - dir := filepath.Dir(path) require.NoError(t, os.MkdirAll(dir, 0775)) require.NoError(t, os.WriteFile(path, []byte(data), 0775)) diff --git a/pkg/flare/archive.go b/pkg/flare/archive.go index c3e5dffe00ca8..8c2ad86688e97 100644 --- a/pkg/flare/archive.go +++ b/pkg/flare/archive.go @@ -214,7 +214,7 @@ func getExpVar(fb flaretypes.FlareBuilder) error { apmDebugPort := pkgconfigsetup.Datadog().GetInt("apm_config.debug.port") f := filepath.Join("expvar", "trace-agent") - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/debug/vars", apmDebugPort)) + resp, err := http.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", apmDebugPort)) if err != nil { return fb.AddFile(f, []byte(fmt.Sprintf("Error retrieving vars: %v", err))) } @@ -394,6 +394,11 @@ func getProcessAgentTaggerList() ([]byte, error) { return nil, fmt.Errorf("wrong configuration to connect to process-agent") } + err = apiutil.SetAuthToken(pkgconfigsetup.Datadog()) + if err != nil { + return nil, err + } + taggerListURL := fmt.Sprintf("http://%s/agent/tagger-list", addressPort) return getTaggerList(taggerListURL) } diff --git a/pkg/flare/archive_linux.go b/pkg/flare/archive_linux.go index dafe8bd41d1bc..9a3aea87a0ac0 100644 --- a/pkg/flare/archive_linux.go +++ b/pkg/flare/archive_linux.go @@ -39,6 +39,7 @@ func addSystemProbePlatformSpecificEntries(fb flaretypes.FlareBuilder) { _ = fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_host.log"), getSystemProbeConntrackHost) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "ebpf_btf_loader.log"), getSystemProbeBTFLoaderInfo) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_sestatus.log"), getSystemProbeSelinuxSestatus) + _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_semodule_list.log"), getSystemProbeSelinuxSemoduleList) } } @@ -155,3 +156,9 @@ func getSystemProbeSelinuxSestatus() ([]byte, error) { url := sysprobeclient.DebugURL("/selinux_sestatus") return getHTTPData(sysProbeClient, url) } + +func getSystemProbeSelinuxSemoduleList() ([]byte, error) { + sysProbeClient := sysprobeclient.Get(getSystemProbeSocketPath()) + url := sysprobeclient.DebugURL("/selinux_semodule_list") + return getHTTPData(sysProbeClient, url) +} diff --git a/pkg/flare/archive_win.go b/pkg/flare/archive_win.go index e62f23d58a8dc..96afe5ac17a94 100644 --- a/pkg/flare/archive_win.go +++ b/pkg/flare/archive_win.go @@ -31,7 +31,7 @@ var ( eventLogChannelsToExport = map[string]string{ "System": "Event/System/Provider[@Name=\"Service Control Manager\"]", - "Application": "Event/System/Provider[@Name=\"datadog-trace-agent\" or @Name=\"DatadogAgent\"]", + "Application": "Event/System/Provider[@Name=\"datadog-trace-agent\" or @Name=\"DatadogAgent\" or @Name=\".NET Runtime\" or @Name=\"Application Error\"]", "Microsoft-Windows-WMI-Activity/Operational": "*", } execTimeout = 30 * time.Second diff --git a/pkg/fleet/daemon/daemon.go b/pkg/fleet/daemon/daemon.go index ba79d5456dd3e..5023393d99e5b 100644 --- a/pkg/fleet/daemon/daemon.go +++ b/pkg/fleet/daemon/daemon.go @@ -570,7 +570,7 @@ type requestState struct { ErrorCode installerErrors.InstallerErrorCode } -func newRequestContext(request remoteAPIRequest) (telemetry.Span, context.Context) { +func newRequestContext(request remoteAPIRequest) (*telemetry.Span, context.Context) { ctx := context.WithValue(context.Background(), requestStateKey, &requestState{ Package: request.Package, ID: request.ID, diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index d9a2a910f45ef..179cad03a2834 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -631,6 +631,8 @@ func (i *installerImpl) promoteExperiment(ctx context.Context, pkg string) error func (i *installerImpl) preparePackage(ctx context.Context, pkg string, _ []string) error { switch pkg { + case packageDatadogInstaller: + return packages.PrepareInstaller(ctx) case packageDatadogAgent: return packages.PrepareAgent(ctx) default: diff --git a/pkg/fleet/installer/oci/download.go b/pkg/fleet/installer/oci/download.go index 7fcce0234512c..e557cf8b53d7f 100644 --- a/pkg/fleet/installer/oci/download.go +++ b/pkg/fleet/installer/oci/download.go @@ -62,8 +62,8 @@ const ( ) const ( - layerMaxSize = 3 << 30 // 3GiB - extractLayerRetries = 3 + layerMaxSize = 3 << 30 // 3GiB + networkRetries = 3 ) var ( @@ -318,30 +318,32 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) return fmt.Errorf("could not get layer media type: %w", err) } if layerMediaType == mediaType { - // Retry stream reset errors - for i := 0; i < extractLayerRetries; i++ { - if i > 0 { - time.Sleep(time.Second) - } - uncompressedLayer, err := layer.Uncompressed() - if err != nil { - return fmt.Errorf("could not uncompress layer: %w", err) - } - err = tar.Extract(uncompressedLayer, dir, layerMaxSize) - uncompressedLayer.Close() - if err != nil { - if !isStreamResetError(err) && !isConnectionResetByPeerError(err) { - return fmt.Errorf("could not extract layer: %w", err) + err = withNetworkRetries( + func() error { + var err error + defer func() { + if err != nil { + deferErr := tar.Clean(dir) + if deferErr != nil { + err = deferErr + } + } + }() + uncompressedLayer, err := layer.Uncompressed() + if err != nil { + return err } - log.Warnf("network error while extracting layer, retrying") - // Clean up the directory before retrying to avoid partial extraction - err = tar.Clean(dir) + err = tar.Extract(uncompressedLayer, dir, layerMaxSize) + uncompressedLayer.Close() if err != nil { - return fmt.Errorf("could not clean directory: %w", err) + return err } - } else { - break - } + + return nil + }, + ) + if err != nil { + return fmt.Errorf("could not extract layer: %w", err) } } } @@ -349,16 +351,22 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) } // WriteOCILayout writes the image as an OCI layout to the given directory. -func (d *DownloadedPackage) WriteOCILayout(dir string) error { - layoutPath, err := layout.Write(dir, empty.Index) - if err != nil { - return fmt.Errorf("could not write layout: %w", err) - } - err = layoutPath.AppendImage(d.Image) - if err != nil { - return fmt.Errorf("could not append image to layout: %w", err) - } - return nil +func (d *DownloadedPackage) WriteOCILayout(dir string) (err error) { + var layoutPath layout.Path + return withNetworkRetries( + func() error { + layoutPath, err = layout.Write(dir, empty.Index) + if err != nil { + return fmt.Errorf("could not write layout: %w", err) + } + + err = layoutPath.AppendImage(d.Image) + if err != nil { + return fmt.Errorf("could not append image to layout: %w", err) + } + return nil + }, + ) } // PackageURL returns the package URL for the given site, package and version. @@ -371,15 +379,50 @@ func PackageURL(env *env.Env, pkg string, version string) string { } } +func withNetworkRetries(f func() error) error { + var err error + for i := 0; i < networkRetries; i++ { + err = f() + if err == nil { + return nil + } + if !isRetryableNetworkError(err) { + return err + } + log.Warnf("retrying after network error: %s", err) + time.Sleep(time.Second) + } + return err +} + +// isRetryableNetworkError returns true if the error is a network error we should retry on +func isRetryableNetworkError(err error) bool { + if err == nil { + return false + } + + if netErr, ok := err.(*net.OpError); ok { + if netErr.Temporary() { + // Temporary errors, such as "connection timed out" + return true + } + if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { + if errno, ok := syscallErr.Err.(syscall.Errno); ok { + // Connection reset errors, such as "connection reset by peer" + return errno == syscall.ECONNRESET + } + } + } + + return isStreamResetError(err) +} + // isStreamResetError returns true if the given error is a stream reset error. // Sometimes, in GCR, the tar extract fails with "stream error: stream ID x; INTERNAL_ERROR; received from peer". // This happens because the uncompressed layer reader is a http/2 response body under the hood. That body is // streamed and receives a "reset stream frame", with the code 0x2 (INTERNAL_ERROR). This is an error from the server // that we need to retry. func isStreamResetError(err error) bool { - if err == nil { - return false - } serr := http2.StreamError{} if errors.As(err, &serr) { return serr.Code == http2.ErrCodeInternal @@ -391,18 +434,6 @@ func isStreamResetError(err error) bool { return false } -// isConnectionResetByPeer returns true if the error is a connection reset by peer error -func isConnectionResetByPeerError(err error) bool { - if netErr, ok := err.(*net.OpError); ok { - if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { - if errno, ok := syscallErr.Err.(syscall.Errno); ok { - return errno == syscall.ECONNRESET - } - } - } - return false -} - type usernamePasswordKeychain struct { username string password string diff --git a/pkg/fleet/installer/packages/datadog_agent.go b/pkg/fleet/installer/packages/datadog_agent.go index c3b56f0e9c87b..7a07b99389ee0 100644 --- a/pkg/fleet/installer/packages/datadog_agent.go +++ b/pkg/fleet/installer/packages/datadog_agent.go @@ -23,7 +23,6 @@ import ( const ( agentPackage = "datadog-agent" - pathOldAgent = "/opt/datadog-agent" agentSymlink = "/usr/bin/datadog-agent" agentUnit = "datadog-agent.service" traceAgentUnit = "datadog-agent-trace.service" @@ -76,13 +75,18 @@ func PrepareAgent(ctx context.Context) (err error) { span, ctx := telemetry.StartSpanFromContext(ctx, "prepare_agent") defer func() { span.Finish(err) }() - // Check if the agent has been installed by a package manager, if yes remove it - if !oldAgentInstalled() { - return nil // Nothing to do - } - err = stopOldAgentUnits(ctx) + err = removeDebRPMPackage(ctx, "datadog-agent") if err != nil { - return fmt.Errorf("failed to stop old agent units: %w", err) + return fmt.Errorf("failed to remove deb/rpm datadog-agent package: %w", err) + } + + for _, unit := range stableUnits { + if err := stopUnit(ctx, unit); err != nil { + log.Warnf("Failed to stop %s: %s", unit, err) + } + if err := disableUnit(ctx, unit); err != nil { + log.Warnf("Failed to disable %s: %s", unit, err) + } } return removeDebRPMPackage(ctx, agentPackage) } @@ -205,28 +209,6 @@ func RemoveAgent(ctx context.Context) error { return nil } -func oldAgentInstalled() bool { - _, err := os.Stat(pathOldAgent) - return err == nil -} - -func stopOldAgentUnits(ctx context.Context) (err error) { - if !oldAgentInstalled() { - return nil - } - span, ctx := telemetry.StartSpanFromContext(ctx, "remove_old_agent_units") - defer span.Finish(err) - for _, unit := range stableUnits { - if err := stopUnit(ctx, unit); err != nil { - return fmt.Errorf("failed to stop %s: %v", unit, err) - } - if err := disableUnit(ctx, unit); err != nil { - return fmt.Errorf("failed to disable %s: %v", unit, err) - } - } - return nil -} - func chownRecursive(path string, uid int, gid int, ignorePaths []string) error { return filepath.Walk(path, func(p string, _ os.FileInfo, err error) error { if err != nil { diff --git a/pkg/fleet/installer/packages/datadog_agent_windows.go b/pkg/fleet/installer/packages/datadog_agent_windows.go index 69935fdee41d8..48e3e1e00148c 100644 --- a/pkg/fleet/installer/packages/datadog_agent_windows.go +++ b/pkg/fleet/installer/packages/datadog_agent_windows.go @@ -10,10 +10,13 @@ package packages import ( "context" "fmt" - + "github.com/DataDog/datadog-agent/pkg/fleet/internal/msi" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" "github.com/DataDog/datadog-agent/pkg/fleet/internal/winregistry" "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" "github.com/DataDog/datadog-agent/pkg/util/log" + "os" + "path" ) const ( @@ -29,9 +32,8 @@ func PrepareAgent(_ context.Context) error { func SetupAgent(ctx context.Context, args []string) (err error) { span, _ := telemetry.StartSpanFromContext(ctx, "setup_agent") defer func() { - if err != nil { - log.Errorf("Failed to setup agent: %s", err) - } + // Don't log error here, or it will appear twice in the output + // since installerImpl.Install will also print the error. span.Finish(err) }() // Make sure there are no Agent already installed @@ -108,20 +110,38 @@ func installAgentPackage(target string, args []string) error { if err != nil { return fmt.Errorf("failed to get Agent user: %w", err) } - args = append(args, fmt.Sprintf("DDAGENTUSER_NAME=%s", agentUser)) - cmd, err := msiexec(target, datadogAgent, "/i", args) + rootPath := "" + _, err = os.Stat(paths.RootTmpDir) + // If bootstrap has not been called before, `paths.RootTmpDir` might not exist + if os.IsExist(err) { + rootPath = paths.RootTmpDir + } + tempDir, err := os.MkdirTemp(rootPath, "datadog-agent") + if err != nil { + return err + } + logFile := path.Join(tempDir, "msi.log") + + cmd, err := msi.Cmd( + msi.Install(), + msi.WithMsiFromPackagePath(target, datadogAgent), + msi.WithDdAgentUserName(agentUser), + msi.WithAdditionalArgs(args), + msi.WithLogFile(path.Join(tempDir, "msi.log")), + ) + var output []byte if err == nil { - err = cmd.Run() + output, err = cmd.Run() } if err != nil { - return fmt.Errorf("failed to install Agent %s: %w", target, err) + return fmt.Errorf("failed to install Agent %s: %w\nLog file located at: %s\n%s", target, err, logFile, string(output)) } return nil } func removeAgentIfInstalled(ctx context.Context) (err error) { - if isProductInstalled("Datadog Agent") { + if msi.IsProductInstalled("Datadog Agent") { span, _ := telemetry.StartSpanFromContext(ctx, "remove_agent") defer func() { if err != nil { @@ -131,7 +151,7 @@ func removeAgentIfInstalled(ctx context.Context) (err error) { } span.Finish(err) }() - err := removeProduct("Datadog Agent") + err := msi.RemoveProduct("Datadog Agent") if err != nil { return err } diff --git a/pkg/fleet/installer/packages/datadog_installer.go b/pkg/fleet/installer/packages/datadog_installer.go index 14d0f042cd0e6..1e36bb462c1fd 100644 --- a/pkg/fleet/installer/packages/datadog_installer.go +++ b/pkg/fleet/installer/packages/datadog_installer.go @@ -41,6 +41,17 @@ func addDDAgentGroup(ctx context.Context) error { return exec.CommandContext(ctx, "groupadd", "--system", "dd-agent").Run() } +// PrepareInstaller prepares the installer +func PrepareInstaller(ctx context.Context) error { + if err := stopUnit(ctx, installerUnit); err != nil { + log.Warnf("Failed to stop unit %s: %s", installerUnit, err) + } + if err := disableUnit(ctx, installerUnit); err != nil { + log.Warnf("Failed to disable %s: %s", installerUnit, err) + } + return nil +} + // SetupInstaller installs and starts the installer systemd units func SetupInstaller(ctx context.Context) (err error) { defer func() { diff --git a/pkg/fleet/installer/packages/datadog_installer_windows.go b/pkg/fleet/installer/packages/datadog_installer_windows.go index d7714ddb257cc..6df65c397dd69 100644 --- a/pkg/fleet/installer/packages/datadog_installer_windows.go +++ b/pkg/fleet/installer/packages/datadog_installer_windows.go @@ -10,78 +10,80 @@ package packages import ( "context" + "fmt" + "os" + "path" - "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" - "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/msi" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" ) const ( datadogInstaller = "datadog-installer" ) +// PrepareInstaller prepares the installer +func PrepareInstaller(_ context.Context) error { + return nil +} + // SetupInstaller installs and starts the installer -func SetupInstaller(ctx context.Context) (err error) { - span, _ := telemetry.StartSpanFromContext(ctx, "setup_installer") - defer func() { - if err != nil { - log.Errorf("Failed to setup installer: %s", err) - } - span.Finish(err) - }() - cmd, err := msiexec("stable", datadogInstaller, "/i", nil) - if err == nil { - // This is the first time that we are installing the installer, - // so we can run it synchronously. - err = cmd.Run() +func SetupInstaller(_ context.Context) error { + rootPath := "" + _, err := os.Stat(paths.RootTmpDir) + // If bootstrap has not been called before, `paths.RootTmpDir` might not exist + if os.IsExist(err) { + rootPath = paths.RootTmpDir + } + tempDir, err := os.MkdirTemp(rootPath, "datadog-installer") + if err != nil { + return err + } + + cmd, err := msi.Cmd(msi.Install(), msi.WithMsiFromPackagePath("stable", datadogInstaller), msi.WithLogFile(path.Join(tempDir, "setup_installer.log"))) + if err != nil { + return fmt.Errorf("failed to setup installer: %w", err) } - return err + output, err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to setup installer: %w\n%s", err, string(output)) + } + return nil } // RemoveInstaller removes the installer -func RemoveInstaller(ctx context.Context) (err error) { - span, _ := telemetry.StartSpanFromContext(ctx, "remove_installer") - defer func() { - if err != nil { - log.Errorf("Failed to remove installer: %s", err) - } - span.Finish(err) - }() - err = removeProduct("Datadog Installer") - return err +func RemoveInstaller(_ context.Context) error { + return msi.RemoveProduct("Datadog Installer") } // StartInstallerExperiment starts the installer experiment -func StartInstallerExperiment(ctx context.Context) (err error) { - span, _ := telemetry.StartSpanFromContext(ctx, "start_installer_experiment") - defer func() { - if err != nil { - log.Errorf("Failed to start installer experiment: %s", err) - } - span.Finish(err) - }() - cmd, err := msiexec("experiment", datadogInstaller, "/i", nil) - if err == nil { - // Launch the msiexec process asynchronously. - err = cmd.Start() +func StartInstallerExperiment(_ context.Context) error { + tempDir, err := os.MkdirTemp(paths.RootTmpDir, "datadog-installer") + if err != nil { + return err + } + + cmd, err := msi.Cmd(msi.Install(), msi.WithMsiFromPackagePath("experiment", datadogInstaller), msi.WithLogFile(path.Join(tempDir, "start_installer_experiment.log"))) + if err != nil { + return fmt.Errorf("failed to start installer experiment: %w", err) } - return err + // Launch the msiexec process asynchronously. + return cmd.FireAndForget() } // StopInstallerExperiment stops the installer experiment -func StopInstallerExperiment(ctx context.Context) (err error) { - span, _ := telemetry.StartSpanFromContext(ctx, "stop_installer_experiment") - defer func() { - if err != nil { - log.Errorf("Failed to stop installer experiment: %s", err) - } - span.Finish(err) - }() - cmd, err := msiexec("stable", datadogInstaller, "/i", nil) - if err == nil { - // Launch the msiexec process asynchronously. - err = cmd.Start() +func StopInstallerExperiment(_ context.Context) error { + tempDir, err := os.MkdirTemp(paths.RootTmpDir, "datadog-installer") + if err != nil { + return err + } + + cmd, err := msi.Cmd(msi.Install(), msi.WithMsiFromPackagePath("stable", datadogInstaller), msi.WithLogFile(path.Join(tempDir, "stop_installer_experiment.log"))) + if err != nil { + return fmt.Errorf("failed to stop installer experiment: %w", err) } - return err + // Launch the msiexec process asynchronously. + return cmd.FireAndForget() } // PromoteInstallerExperiment promotes the installer experiment diff --git a/pkg/fleet/installer/packages/pkg_manager.go b/pkg/fleet/installer/packages/pkg_manager.go index 98b9d76b29d59..90eb34a5b2f52 100644 --- a/pkg/fleet/installer/packages/pkg_manager.go +++ b/pkg/fleet/installer/packages/pkg_manager.go @@ -8,44 +8,62 @@ package packages import ( - "bytes" "context" + "errors" "fmt" "os/exec" "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" ) +func dpkgInstalled() (bool, error) { + _, err := exec.LookPath("dpkg") + if err != nil && !errors.Is(err, exec.ErrNotFound) { + return false, err + } + return err == nil, nil +} + +func rpmInstalled() (bool, error) { + _, err := exec.LookPath("rpm") + if err != nil && !errors.Is(err, exec.ErrNotFound) { + return false, err + } + return err == nil, nil +} + // removeDebRPMPackage removes a package installed via deb/rpm package manager // It doesn't remove dependencies or purge as we want to keep existing configuration files // and reinstall the package using the installer. // Note: we don't run the pre/post remove scripts as we want to avoid surprises for older agent versions (like removing config) func removeDebRPMPackage(ctx context.Context, pkg string) (err error) { - span, _ := telemetry.StartSpanFromContext(ctx, "remove_deb_rpm_package") + span, _ := telemetry.StartSpanFromContext(ctx, "removeDebRPMPackage") defer func() { span.Finish(err) }() - // Compute the right command depending on the package manager - var cmd *exec.Cmd - if _, pathErr := exec.LookPath("dpkg"); pathErr == nil { - // Doesn't fail if the package isn't installed - cmd = exec.Command("dpkg", "-r", "--no-triggers", agentPackage) - } else if _, pathErr := exec.LookPath("rpm"); pathErr == nil { - // Check if package exist, else the command will fail - pkgErr := exec.Command("rpm", "-q", agentPackage).Run() - if pkgErr == nil { - cmd = exec.Command("rpm", "-e", "--nodeps", "--noscripts", agentPackage) - } - } - - if cmd == nil { - // If we can't find a package manager or the package is not installed, ignore this step + + dpkgInstalled, err := dpkgInstalled() + if err != nil { + return err + } + rpmInstalled, err := rpmInstalled() + if err != nil { + return err + } + var packageInstalled bool + var removeCmd *exec.Cmd + if dpkgInstalled { + removeCmd = exec.Command("dpkg", "-r", pkg) + packageInstalled = exec.Command("dpkg", "-s", pkg).Run() == nil + } + if rpmInstalled { + removeCmd = exec.Command("rpm", "-e", pkg) + packageInstalled = exec.Command("rpm", "-q", pkg).Run() == nil + } + if !packageInstalled { return nil } - - // Run the command - var buf bytes.Buffer - cmd.Stderr = &buf - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to uninstall deb/rpm package %s (%w): %s", pkg, err, buf.String()) + out, err := removeCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to uninstall deb/rpm package %s (%w): %s", pkg, err, out) } return nil } diff --git a/pkg/fleet/installer/setup/common/config.go b/pkg/fleet/installer/setup/common/config.go index 650b6b0acebe7..7d9e00aba1483 100644 --- a/pkg/fleet/installer/setup/common/config.go +++ b/pkg/fleet/installer/setup/common/config.go @@ -103,6 +103,7 @@ type DatadogConfig struct { APIKey string `yaml:"api_key"` Hostname string `yaml:"hostname,omitempty"` Site string `yaml:"site,omitempty"` + Proxy DatadogConfigProxy `yaml:"proxy,omitempty"` Env string `yaml:"env,omitempty"` Tags []string `yaml:"tags,omitempty"` LogsEnabled bool `yaml:"logs_enabled,omitempty"` @@ -111,6 +112,13 @@ type DatadogConfig struct { ExpectedTagsDuration string `yaml:"expected_tags_duration,omitempty"` } +// DatadogConfigProxy represents the configuration for the proxy +type DatadogConfigProxy struct { + HTTP string `yaml:"http,omitempty"` + HTTPS string `yaml:"https,omitempty"` + NoProxy []string `yaml:"no_proxy,omitempty"` +} + // DatadogConfigDJM represents the configuration for the Data Jobs Monitoring type DatadogConfigDJM struct { Enabled bool `yaml:"enabled,omitempty"` diff --git a/pkg/fleet/installer/setup/common/setup.go b/pkg/fleet/installer/setup/common/setup.go index 60931ef01da69..b6e512d7dc5d6 100644 --- a/pkg/fleet/installer/setup/common/setup.go +++ b/pkg/fleet/installer/setup/common/setup.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "os" + "strings" "time" "github.com/DataDog/datadog-agent/pkg/fleet/installer" @@ -40,7 +41,7 @@ type Setup struct { Out *Output Env *env.Env Ctx context.Context - Span telemetry.Span + Span *telemetry.Span Packages Packages Config Config } @@ -60,6 +61,10 @@ Running the %s installation script (https://github.com/DataDog/datadog-agent/tre if err != nil { return nil, fmt.Errorf("failed to create installer: %w", err) } + var proxyNoProxy []string + if os.Getenv("DD_PROXY_NO_PROXY") != "" { + proxyNoProxy = strings.Split(os.Getenv("DD_PROXY_NO_PROXY"), ",") + } span, ctx := telemetry.StartSpanFromContext(ctx, fmt.Sprintf("setup.%s", flavor)) s := &Setup{ configDir: configDir, @@ -75,7 +80,12 @@ Running the %s installation script (https://github.com/DataDog/datadog-agent/tre APIKey: env.APIKey, Hostname: os.Getenv("DD_HOSTNAME"), Site: env.Site, - Env: os.Getenv("DD_ENV"), + Proxy: DatadogConfigProxy{ + HTTP: os.Getenv("DD_PROXY_HTTP"), + HTTPS: os.Getenv("DD_PROXY_HTTPS"), + NoProxy: proxyNoProxy, + }, + Env: os.Getenv("DD_ENV"), }, IntegrationConfigs: make(map[string]IntegrationConfig), }, @@ -120,7 +130,7 @@ func (s *Setup) installPackage(name string, url string) (err error) { span, ctx := telemetry.StartSpanFromContext(s.Ctx, "install") defer func() { span.Finish(err) }() span.SetTag("url", url) - span.SetTag("_top_level", 1) + span.SetTopLevel() s.Out.WriteString(fmt.Sprintf("Installing %s...\n", name)) err = s.installer.Install(ctx, url, nil) diff --git a/pkg/fleet/installer/setup/djm/databricks.go b/pkg/fleet/installer/setup/djm/databricks.go index af54e1bb99553..e1e699f65a2de 100644 --- a/pkg/fleet/installer/setup/djm/databricks.go +++ b/pkg/fleet/installer/setup/djm/databricks.go @@ -17,9 +17,9 @@ import ( ) const ( - databricksInjectorVersion = "0.26.0-1" - databricksJavaVersion = "1.42.2-1" - databricksAgentVersion = "7.58.2-1" + databricksInjectorVersion = "0.26.0-1" + databricksJavaTracerVersion = "1.42.2-1" + databricksAgentVersion = "7.58.2-1" ) var ( @@ -65,7 +65,7 @@ var ( Service: "databricks", }, } - tracerEnvConfig = []common.InjectTracerConfigEnvVar{ + tracerEnvConfigDatabricks = []common.InjectTracerConfigEnvVar{ { Key: "DD_DATA_JOBS_ENABLED", Value: "true", @@ -80,6 +80,8 @@ var ( // SetupDatabricks sets up the Databricks environment func SetupDatabricks(s *common.Setup) error { s.Packages.Install(common.DatadogAgentPackage, databricksAgentVersion) + s.Packages.Install(common.DatadogAPMInjectPackage, databricksInjectorVersion) + s.Packages.Install(common.DatadogAPMLibraryJavaPackage, databricksJavaTracerVersion) hostname, err := os.Hostname() if err != nil { @@ -90,6 +92,8 @@ func SetupDatabricks(s *common.Setup) error { s.Config.DatadogYAML.ExpectedTagsDuration = "10m" s.Config.DatadogYAML.ProcessConfig.ExpvarPort = 6063 // avoid port conflict on 6062 + s.Config.InjectTracerYAML.AdditionalEnvironmentVariables = tracerEnvConfigDatabricks + setupCommonHostTags(s) installMethod := "manual" if os.Getenv("DD_DJM_INIT_IS_MANAGED_INSTALL") != "true" { @@ -168,11 +172,7 @@ func setHostTag(s *common.Setup, tagKey, value string) { func setupDatabricksDriver(s *common.Setup) { s.Span.SetTag("spark_node", "driver") - s.Packages.Install(common.DatadogAPMInjectPackage, databricksInjectorVersion) - s.Packages.Install(common.DatadogAPMLibraryJavaPackage, databricksJavaVersion) - s.Config.DatadogYAML.Tags = append(s.Config.DatadogYAML.Tags, "node_type:driver") - s.Config.InjectTracerYAML.AdditionalEnvironmentVariables = tracerEnvConfig var sparkIntegration common.IntegrationConfig if os.Getenv("DRIVER_LOGS_ENABLED") == "true" { @@ -197,8 +197,6 @@ func setupDatabricksDriver(s *common.Setup) { func setupDatabricksWorker(s *common.Setup) { s.Span.SetTag("spark_node", "worker") - s.Packages.Install(common.DatadogAgentPackage, databricksAgentVersion) - s.Config.DatadogYAML.Tags = append(s.Config.DatadogYAML.Tags, "node_type:worker") var sparkIntegration common.IntegrationConfig diff --git a/pkg/fleet/internal/bootstrap/bootstrap_windows.go b/pkg/fleet/internal/bootstrap/bootstrap_windows.go index 6990ec77cf042..7172d02a1b6a9 100644 --- a/pkg/fleet/internal/bootstrap/bootstrap_windows.go +++ b/pkg/fleet/internal/bootstrap/bootstrap_windows.go @@ -11,8 +11,8 @@ package bootstrap import ( "context" "fmt" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/msi" "os" - "os/exec" "path/filepath" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" @@ -81,20 +81,19 @@ func downloadInstaller(ctx context.Context, env *env.Env, url string, tmpDir str } else if len(msis) == 0 { return nil, fmt.Errorf("no MSIs in package") } - msiArgs := []string{ - "/i", - msis[0], - "/qn", - "MSIFASTINSTALL=7", - } - if env.AgentUserName != "" { - msiArgs = append(msiArgs, fmt.Sprintf("DDAGENTUSER_NAME=%s", env.AgentUserName)) - // don't need to look at the registry here since the installer will read it if the command line - // parameter is not provided + + cmd, err := msi.Cmd( + msi.Install(), + msi.WithMsi(msis[0]), + msi.WithDdAgentUserName(env.AgentUserName), + ) + var output []byte + if err == nil { + output, err = cmd.Run() } - err = exec.Command("msiexec", msiArgs...).Run() + if err != nil { - return nil, fmt.Errorf("failed to install the Datadog Installer") + return nil, fmt.Errorf("failed to install the Datadog Installer: %w\n%s", err, string(output)) } return iexec.NewInstallerExec(env, paths.StableInstallerPath), nil } diff --git a/pkg/fleet/internal/cdn/cdn.go b/pkg/fleet/internal/cdn/cdn.go index f35418cc0b7bc..dcb57bf0ee17d 100644 --- a/pkg/fleet/internal/cdn/cdn.go +++ b/pkg/fleet/internal/cdn/cdn.go @@ -165,28 +165,17 @@ func (c *CDN) Close() error { } // writePolicyMetadata writes the policy metadata to the given directory -// and makes it readable to dd-agent +// and makes it world-readable func writePolicyMetadata(config Config, dir string) error { - ddAgentUID, ddAgentGID, err := getAgentIDs() - if err != nil { - return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) - } - state := config.State() stateBytes, err := json.Marshal(state) if err != nil { return fmt.Errorf("could not marshal state: %w", err) } - err = os.WriteFile(filepath.Join(dir, policyMetadataFilename), stateBytes, 0440) + err = os.WriteFile(filepath.Join(dir, policyMetadataFilename), stateBytes, 0444) if err != nil { return fmt.Errorf("could not write %s: %w", policyMetadataFilename, err) } - if runtime.GOOS != "windows" { - err = os.Chown(filepath.Join(dir, policyMetadataFilename), ddAgentUID, ddAgentGID) - if err != nil { - return fmt.Errorf("could not chown %s: %w", policyMetadataFilename, err) - } - } return nil } diff --git a/pkg/fleet/internal/cdn/scope_expression.go b/pkg/fleet/internal/cdn/scope_expression.go index 659d6798c0670..cc86626fcf9c0 100644 --- a/pkg/fleet/internal/cdn/scope_expression.go +++ b/pkg/fleet/internal/cdn/scope_expression.go @@ -10,7 +10,9 @@ import ( "fmt" "regexp" + "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/utils" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "github.com/DataDog/datadog-agent/pkg/gohai/platform" "github.com/DataDog/datadog-agent/pkg/version" "github.com/expr-lang/expr" ) @@ -115,6 +117,8 @@ func getScopeExprVars(env *env.Env, hostTagsGetter hostTagsGetter) map[string]in return map[string]interface{}{ "hostname": env.Hostname, "installer_version": version.AgentVersion, // AgentVersion evaluates to the installer version here + "os": platform.CollectInfo().KernelName.ValueOrDefault(), + "os_version": utils.GetOSVersion(), "tags": hostTagsGetter.get(), } diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index 834f54b3d8a53..92f5880723fa9 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -40,7 +40,7 @@ func NewInstallerExec(env *env.Env, installerBinPath string) *InstallerExec { type installerCmd struct { *exec.Cmd - span telemetry.Span + span *telemetry.Span ctx context.Context } diff --git a/pkg/fleet/internal/msi/file_in_use.log b/pkg/fleet/internal/msi/file_in_use.log new file mode 100644 index 0000000000000..eb0ad60b3ddd6 Binary files /dev/null and b/pkg/fleet/internal/msi/file_in_use.log differ diff --git a/pkg/fleet/internal/msi/invalid_credentials.log b/pkg/fleet/internal/msi/invalid_credentials.log new file mode 100644 index 0000000000000..3d568391bdb12 Binary files /dev/null and b/pkg/fleet/internal/msi/invalid_credentials.log differ diff --git a/pkg/fleet/internal/msi/missing_password_for_dc.log b/pkg/fleet/internal/msi/missing_password_for_dc.log new file mode 100644 index 0000000000000..c87c3dc183493 Binary files /dev/null and b/pkg/fleet/internal/msi/missing_password_for_dc.log differ diff --git a/pkg/fleet/internal/msi/msiexec.go b/pkg/fleet/internal/msi/msiexec.go new file mode 100644 index 0000000000000..f2ca387fb3aa2 --- /dev/null +++ b/pkg/fleet/internal/msi/msiexec.go @@ -0,0 +1,289 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build windows + +// Package msi contains helper functions to work with msi packages +package msi + +import ( + "errors" + "fmt" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" +) + +type msiexecArgs struct { + // target should be either a full path to a MSI, an URL to a MSI or a product code. + target string + + // msiAction should be "/i" for installation, "/x" for uninstallation etc... + msiAction string + + // logFile should be a full local path where msiexec will write the installation logs. + // If nothing is specified, a random, temporary file is used. + logFile string + ddagentUserName string + + // additionalArgs are further args that can be passed to msiexec + additionalArgs []string +} + +// MsiexecOption is an option type for creating msiexec command lines +type MsiexecOption func(*msiexecArgs) error + +// Install specifies that msiexec will be invoked to install a product +func Install() MsiexecOption { + return func(a *msiexecArgs) error { + a.msiAction = "/i" + return nil + } +} + +// Uninstall specifies that msiexec will be invoked to uninstall a product +func Uninstall() MsiexecOption { + return func(a *msiexecArgs) error { + a.msiAction = "/x" + return nil + } +} + +// WithMsi specifies the MSI target for msiexec +func WithMsi(target string) MsiexecOption { + return func(a *msiexecArgs) error { + a.target = target + return nil + } +} + +// WithMsiFromPackagePath finds an MSI from the packages folder +func WithMsiFromPackagePath(target, product string) MsiexecOption { + return func(a *msiexecArgs) error { + updaterPath := filepath.Join(paths.PackagesPath, product, target) + msis, err := filepath.Glob(filepath.Join(updaterPath, fmt.Sprintf("%s-*-1-x86_64.msi", product))) + if err != nil { + return err + } + if len(msis) > 1 { + return fmt.Errorf("too many MSIs in package") + } else if len(msis) == 0 { + return fmt.Errorf("no MSIs in package") + } + a.target = msis[0] + return nil + } +} + +// WithProduct specifies the product name to target for msiexec +func WithProduct(productName string) MsiexecOption { + return func(a *msiexecArgs) error { + product, err := FindProductCode(productName) + if err != nil { + return fmt.Errorf("error trying to find product %s: %w", productName, err) + } + a.target = product.Code + return nil + } +} + +// WithLogFile specifies the log file for msiexec +func WithLogFile(logFile string) MsiexecOption { + return func(a *msiexecArgs) error { + a.logFile = logFile + return nil + } +} + +// WithAdditionalArgs specifies additional arguments for msiexec +func WithAdditionalArgs(additionalArgs []string) MsiexecOption { + return func(a *msiexecArgs) error { + a.additionalArgs = additionalArgs + return nil + } +} + +// WithDdAgentUserName specifies the DDAGENTUSER_NAME to use +func WithDdAgentUserName(ddagentUserName string) MsiexecOption { + return func(a *msiexecArgs) error { + a.ddagentUserName = ddagentUserName + return nil + } +} + +// Msiexec is a type wrapping msiexec +type Msiexec struct { + *exec.Cmd + + // logFile is the path to the MSI log file + logFile string + + // postExecActions is a list of actions to be executed after msiexec has run + postExecActions []func() +} + +func (m *Msiexec) openAndProcessLogFile() ([]byte, error) { + logfile, err := os.Open(m.logFile) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + // File does not exist is not necessarily an error + return nil, nil + } + return nil, err + } + result, err := m.processLogFile(logfile) + _ = logfile.Close() + return result, err +} + +// processLogFile takes an open file and processes it with a series of processors to obtain +// a condensed version of the log file with only the relevant information. +func (m *Msiexec) processLogFile(logFile fs.File) ([]byte, error) { + // Compile a list of regular expressions we are interested in extracting from the logs + return processLogFile(logFile, + func(bytes []byte) []TextRange { + // Only need one TextRange of context before and after since other regexes will combine + return FindAllIndexWithContext(regexp.MustCompile("Datadog[.]CustomActions.*"), bytes, 1, 1) + }, + func(bytes []byte) []TextRange { + // Only need one TextRange of context before and after since other regexes will combine + return FindAllIndexWithContext(regexp.MustCompile("System[.]Exception"), bytes, 1, 1) + }, + func(bytes []byte) []TextRange { + // typically looks like this: + // Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.StartDDServices + // CA: 01:50:49: StartDDServices. Failed to start services: System.InvalidOperationException: Cannot start service datadogagent on computer '.'. ---> System.ComponentModel.Win32Exception: The service did not start due to a logon failure + // --- End of inner exception stack trace --- + // at System.ServiceProcess.ServiceController.Start(String args) + // at Datadog.CustomActions.Native.ServiceController.StartService(String serviceName, TimeSpan timeout) + // at Datadog.CustomActions.ServiceCustomAction.StartDDServices() + // Other regexes will pick up on the stack trace, but there's not much information to get before the error + return FindAllIndexWithContext(regexp.MustCompile("Cannot start service"), bytes, 1, 2) + }, + func(bytes []byte) []TextRange { + // Typically looks like this: + // CA(ddnpm): DriverInstall: serviceDef::create() + // CA(ddnpm): DriverInstall: Failed to CreateService 1073 + // CA(ddnpm): DriverInstall: Service exists, verifying + // CA(ddnpm): DriverInstall: Updated path for existing service + // So include a bit of context before and after + return FindAllIndexWithContext(regexp.MustCompile("Failed to CreateService"), bytes, 5, 5) + }, + func(bytes []byte) []TextRange { + // Typically looks like this: + // Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.ProcessDdAgentUserCredentials + // CA: 01:49:43: LookupAccountWithExtendedDomainSyntax. User not found, trying again with fixed domain part: \toto + // CA: 01:49:43: ProcessDdAgentUserCredentials. User toto doesn't exist. + // CA: 01:49:43: ProcessDdAgentUserCredentials. domain part is empty, using default + // CA: 01:49:43: ProcessDdAgentUserCredentials. Installing with DDAGENTUSER_PROCESSED_NAME=toto and DDAGENTUSER_PROCESSED_DOMAIN=datadoghq-qa-labs.local + // CA: 01:49:43: HandleProcessDdAgentUserCredentialsException. Error processing ddAgentUser credentials: Datadog.CustomActions.InvalidAgentUserConfigurationException: A password was not provided. A password is a required when installing on Domain Controllers. + // at Datadog.CustomActions.ProcessUserCustomActions.ProcessDdAgentUserCredentials(Boolean calledFromUIControl) + // MSI (s) (C8!50) [01:49:43:906]: Product: Datadog Agent -- A password was not provided. A password is a required when installing on Domain Controllers. + // + // A password was not provided. A password is a required when installing on Domain Controllers. + // CustomAction ProcessDdAgentUserCredentials returned actual error code 1603 (note this may not be 100% accurate if translation happened inside sandbox) + // Action ended 1:49:43: ProcessDdAgentUserCredentials. Return value 3. + // So include lots of context to ensure we get the full picture + return FindAllIndexWithContext(regexp.MustCompile("A password was not provided"), bytes, 6, 6) + }, + func(bytes []byte) []TextRange { + // Typically looks like this: + // Info 1603. The file C:\Program Files\Datadog\Datadog Agent\bin\agent\process-agent.exe is being held in use by the following process: Name: process-agent, Id: 4704, Window Title: '(not determined yet)'. Close that application and retry. + // Not much context to be had before and after + return FindAllIndexWithContext(regexp.MustCompile("is being held in use by the following process"), bytes, 1, 1) + }, + func(bytes []byte) []TextRange { + // Typically looks like this: + // Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.StartDDServices + // CustomAction WixFailWhenDeferred returned actual error code 1603 (note this may not be 100% accurate if translation happened inside sandbox) + // Action ended 2:11:49: InstallFinalize. Return value 3. + // The important context is the TextRange after the error ("Return value 3") but the previous lines can include some useful information too + return FindAllIndexWithContext(regexp.MustCompile("returned actual error"), bytes, 5, 1) + }, + func(bytes []byte) []TextRange { + // Typically looks like this: + // Action 12:24:00: InstallServices. Installing new services + // InstallServices: Service: + // Error 1923. Service 'Datadog Agent' (datadogagent) could not be installed. Verify that you have sufficient privileges to install system services. + // MSI (s) (54:EC) [12:25:53:886]: Product: Datadog Agent -- Error 1923. Service 'Datadog Agent' (datadogagent) could not be installed. Verify that you have sufficient privileges to install system services. + return FindAllIndexWithContext(regexp.MustCompile("Verify that you have sufficient privileges to install system services"), bytes, 2, 1) + }) +} + +// Run runs msiexec synchronously +func (m *Msiexec) Run() ([]byte, error) { + err := m.Cmd.Run() + // The log file *should not* be too big. Avoid verbose log files. + logFileBytes, err2 := m.openAndProcessLogFile() + err = errors.Join(err, err2) + for _, p := range m.postExecActions { + p() + } + + return logFileBytes, err +} + +// RunAsync runs msiexec asynchronously +func (m *Msiexec) RunAsync(done func([]byte, error)) error { + err := m.Cmd.Start() + if err != nil { + return err + } + go func() { + err := m.Cmd.Wait() + // The log file *should not* be too big. Avoid verbose log files. + logFileBytes, err2 := m.openAndProcessLogFile() + err = errors.Join(err, err2) + for _, p := range m.postExecActions { + p() + } + done(logFileBytes, err) + }() + return nil +} + +// FireAndForget starts msiexec and doesn't wait for it to finish. +// The log file won't be read at the end and post execution actions will not be executed. +func (m *Msiexec) FireAndForget() error { + return m.Cmd.Start() +} + +// Cmd creates a new Msiexec wrapper around cmd.Exec that will call msiexec +func Cmd(options ...MsiexecOption) (*Msiexec, error) { + a := &msiexecArgs{} + for _, opt := range options { + if err := opt(a); err != nil { + return nil, err + } + } + if a.msiAction == "" || a.target == "" { + return nil, fmt.Errorf("argument error") + } + + cmd := &Msiexec{} + if len(a.logFile) == 0 { + tempDir, err := os.MkdirTemp("", "datadog-installer") + if err != nil { + return nil, err + } + a.logFile = path.Join(tempDir, "msi.log") + cmd.postExecActions = append(cmd.postExecActions, func() { + _ = os.RemoveAll(tempDir) + }) + } + args := append(a.additionalArgs, a.msiAction, a.target, "/qn", "MSIFASTINSTALL=7", "/log", a.logFile) + if a.ddagentUserName != "" { + args = append(args, fmt.Sprintf("DDAGENTUSER_NAME=%s", a.ddagentUserName)) + } + + cmd.Cmd = exec.Command("msiexec", args...) + cmd.logFile = a.logFile + + return cmd, nil +} diff --git a/pkg/fleet/internal/msi/msilog.go b/pkg/fleet/internal/msi/msilog.go new file mode 100644 index 0000000000000..47f8ef5f30641 --- /dev/null +++ b/pkg/fleet/internal/msi/msilog.go @@ -0,0 +1,130 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build windows + +// Package msi contains helper functions to work with msi packages +package msi + +import ( + "bytes" + "fmt" + "golang.org/x/text/encoding/unicode" + "io" + "io/fs" + "regexp" + "sort" +) + +// TextRange is a simple struct to represent a range of text in a file. +type TextRange struct { + start int + end int +} + +// FindAllIndexWithContext is similar to FindAllIndex but expands the matched range for a number of lines +// before and after the TextRange (called contextBefore and contextAfter). +func FindAllIndexWithContext(r *regexp.Regexp, input []byte, contextBefore, contextAfter int) []TextRange { + contextBefore = max(contextBefore, 0) + contextAfter = max(contextAfter, 0) + var extractedRanges []TextRange + results := r.FindAllIndex(input, -1) + for _, result := range results { + lineCounter := 0 + charCounter := result[0] + for ; charCounter >= 0; charCounter-- { + if input[charCounter] == '\n' { + lineCounter++ + } + if lineCounter > contextBefore { + break + } + } + lineStart := charCounter + 1 + + lineCounter = 0 + charCounter = result[1] + for ; charCounter < len(input); charCounter++ { + if input[charCounter] == '\n' { + lineCounter++ + } + if lineCounter > contextAfter { + break + } + } + lineEnd := charCounter + + extractedRanges = append(extractedRanges, TextRange{lineStart, lineEnd}) + } + + return extractedRanges +} + +// insert merges newRanges into existingRanges by combining overlapping or adjacent ranges. +func insert(existingRanges, newRanges []TextRange) []TextRange { + // Combine all ranges into a single slice for sorting + allRanges := append(existingRanges, newRanges...) + + // Sort ranges by start value (and end value if starts are equal) + sort.Slice(allRanges, func(i, j int) bool { + if allRanges[i].start == allRanges[j].start { + return allRanges[i].end < allRanges[j].end + } + return allRanges[i].start < allRanges[j].start + }) + + // Merge ranges + var merged []TextRange + for _, current := range allRanges { + // If merged is empty or the current range does not overlap with the last merged range + if len(merged) == 0 || merged[len(merged)-1].end < current.start { + merged = append(merged, current) // Add the current range + } else { + // Overlapping or adjacent: Extend the end of the last merged range + merged[len(merged)-1].end = max(merged[len(merged)-1].end, current.end) + } + } + + return merged +} + +// Combine processes input using multiple logFileProcessors and merges their output ranges. +func Combine(input []byte, processors ...logFileProcessor) []TextRange { + var allRanges []TextRange + + // Collect all ranges from each processor + for _, processor := range processors { + allRanges = append(allRanges, processor(input)...) + } + + // Use the improved insert function to merge all collected ranges + return insert(nil, allRanges) +} + +type logFileProcessor func([]byte) []TextRange + +// processLogFile reads a UTF-16 MSI log file and applies various processors on it +// to retain only the relevant log lines. It combines the various outputs from the processors and +// decorate each range of log lines with a marker ('---') to distinguish them. +func processLogFile(logFile fs.File, processors ...logFileProcessor) ([]byte, error) { + logFileBuffer := bytes.NewBuffer(nil) + _, err := io.Copy(logFileBuffer, logFile) + if err != nil { + return nil, err + } + decodedLogsBytes, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewDecoder().Bytes(logFileBuffer.Bytes()) + if err != nil { + return nil, err + } + + var output []byte + rangesToSave := Combine(decodedLogsBytes, processors...) + for _, ranges := range rangesToSave { + output = append(output, []byte(fmt.Sprintf("--- %d:%d\r\n", ranges.start, ranges.end))...) + output = append(output, decodedLogsBytes[ranges.start:ranges.end]...) + output = append(output, '\r', '\n', '\r', '\n') + } + return output, nil +} diff --git a/pkg/fleet/internal/msi/msilog_test.go b/pkg/fleet/internal/msi/msilog_test.go new file mode 100644 index 0000000000000..f52ff24f2a65f --- /dev/null +++ b/pkg/fleet/internal/msi/msilog_test.go @@ -0,0 +1,393 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build windows + +package msi + +import ( + "embed" + "github.com/stretchr/testify/require" + "golang.org/x/text/encoding/unicode" + "regexp" + "strings" + "testing" +) + +//go:embed wixfailwhendeferred.log +//go:embed missing_password_for_dc.log +//go:embed file_in_use.log +//go:embed invalid_credentials.log +//go:embed service_marked_for_deletion.log +var logFilesFS embed.FS + +func TestFindAllIndexWithContext(t *testing.T) { + data, err := logFilesFS.ReadFile("wixfailwhendeferred.log") + require.NoError(t, err) + + r := regexp.MustCompile("returned actual error") + decodedLogsBytes, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewDecoder().Bytes(data) + require.NoError(t, err) + + matches := FindAllIndexWithContext(r, decodedLogsBytes, 2, 1) + + expectedMatches := []string{ + `Action start 2:10:53: WixRemoveFoldersEx. +WixRemoveFoldersEx: Error 0x80070057: Missing folder property: dd_PROJECTLOCATION_0 for row: RemoveFolderEx +CustomAction WixRemoveFoldersEx returned actual error code 1603 but will be translated to success due to continue marking +Action ended 2:10:53: WixRemoveFoldersEx. Return value 1.`, + `Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.PrepareDecompressPythonDistributions +CA: 02:10:56: PrepareDecompressPythonDistributions. Could not set the progress bar size +CustomAction PrepareDecompressPythonDistributions returned actual error code 1603 but will be translated to success due to continue marking +Action ended 2:10:56: PrepareDecompressPythonDistributions. Return value 1.`, + `SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.StartDDServices +CustomAction WixFailWhenDeferred returned actual error code 1603 (note this may not be 100% accurate if translation happened inside sandbox) +Action ended 2:11:49: InstallFinalize. Return value 3.`, + } + for i, expectedMatch := range expectedMatches { + text := strings.ReplaceAll(string(decodedLogsBytes[matches[i].start:matches[i].end]), "\r", "") + require.Equal(t, expectedMatch, text) + } +} + +func TestFindAllIndexWithContextOutOfRangeDoesntFail(t *testing.T) { + data, err := logFilesFS.ReadFile("wixfailwhendeferred.log") + require.NoError(t, err) + + r := regexp.MustCompile("returned actual error") + decodedLogsBytes, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewDecoder().Bytes(data) + require.NoError(t, err) + + matches := FindAllIndexWithContext(r, decodedLogsBytes, -21, -1435) + expectedMatches := []string{ + `CustomAction WixRemoveFoldersEx returned actual error code 1603 but will be translated to success due to continue marking`, + `CustomAction PrepareDecompressPythonDistributions returned actual error code 1603 but will be translated to success due to continue marking`, + `CustomAction WixFailWhenDeferred returned actual error code 1603 (note this may not be 100% accurate if translation happened inside sandbox)`, + } + for i, expectedMatch := range expectedMatches { + text := strings.ReplaceAll(string(decodedLogsBytes[matches[i].start:matches[i].end]), "\r", "") + require.Equal(t, expectedMatch, text) + } +} + +func TestCombineRanges(t *testing.T) { + tests := map[string]struct { + input []logFileProcessor + expected []TextRange + }{ + "overlap left": { + input: []logFileProcessor{ + func(_ []byte) []TextRange { + return []TextRange{{1, 5}} + }, + func(_ []byte) []TextRange { + return []TextRange{{4, 7}} + }}, + expected: []TextRange{{1, 7}}, + }, + "overlap right": { + input: []logFileProcessor{ + func(_ []byte) []TextRange { + return []TextRange{{2, 4}} + }, + func(_ []byte) []TextRange { + return []TextRange{{1, 3}} + }}, + expected: []TextRange{{1, 4}}, + }, + "no overlap": { + input: []logFileProcessor{ + func(_ []byte) []TextRange { + return []TextRange{{2, 4}} + }, + func(_ []byte) []TextRange { + return []TextRange{{5, 10}} + }}, + expected: []TextRange{{2, 4}, {5, 10}}, + }, + "full overlap": { + input: []logFileProcessor{ + func(_ []byte) []TextRange { + return []TextRange{{2, 4}} + }, + func(_ []byte) []TextRange { + return []TextRange{{1, 10}} + }}, + expected: []TextRange{{1, 10}}, + }, + "full overlap inverted": { + input: []logFileProcessor{ + func(_ []byte) []TextRange { + return []TextRange{{1, 10}} + }, + func(_ []byte) []TextRange { + return []TextRange{{2, 4}} + }}, + expected: []TextRange{{1, 10}}, + }, + "test many ranges": { + input: []logFileProcessor{ + func(_ []byte) []TextRange { + return []TextRange{{16067, 16421}} + }, + func(_ []byte) []TextRange { + return []TextRange{{19659, 20140}} + }, + func(_ []byte) []TextRange { + return []TextRange{{16002, 16359}} + }, + func(_ []byte) []TextRange { + return []TextRange{{19559, 19951}} + }, + func(_ []byte) []TextRange { + return []TextRange{{59421, 59556}} + }}, + expected: []TextRange{{16002, 16421}, {19559, 20140}, {59421, 59556}}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + require.Equal(t, test.expected, Combine(nil, test.input...)) + }) + } +} + +func TestReadLogFile(t *testing.T) { + m := &Msiexec{} + + tests := map[string]struct { + input string + expected string + }{ + "Wix built-in failure mode": { + input: "wixfailwhendeferred.log", + expected: `--- 1547:2038 +Action ended 2:10:53: LaunchConditions. Return value 1. +Action start 2:10:53: ValidateProductID. +Action ended 2:10:53: ValidateProductID. Return value 1. +Action start 2:10:53: WixRemoveFoldersEx. +WixRemoveFoldersEx: Error 0x80070057: Missing folder property: dd_PROJECTLOCATION_0 for row: RemoveFolderEx +CustomAction WixRemoveFoldersEx returned actual error code 1603 but will be translated to success due to continue marking +Action ended 2:10:53: WixRemoveFoldersEx. Return value 1. + +--- 6770:7391 +Action start 2:10:55: PrepareDecompressPythonDistributions. +SFXCA: Extracting custom action to temporary directory: C:\Windows\Installer\MSIB32B.tmp-\ +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.PrepareDecompressPythonDistributions +CA: 02:10:56: PrepareDecompressPythonDistributions. Could not set the progress bar size +CustomAction PrepareDecompressPythonDistributions returned actual error code 1603 but will be translated to success due to continue marking +Action ended 2:10:56: PrepareDecompressPythonDistributions. Return value 1. + +--- 16002:16421 +CA: 02:11:30: AddUser. ddagentuser already exists, not creating +CA: 02:11:30: GetPreviousAgentUser. Could not find previous agent user: System.Exception: Agent user information is not in registry + at Datadog.CustomActions.InstallStateCustomActions.GetPreviousAgentUser(ISession session, IRegistryServices registryServices, INativeMethods nativeMethods) +CA: 02:11:30: ConfigureUser. Resetting ddagentuser password. + +--- 19559:20140 +CA: 02:11:36: ConfigureServiceUsers. Configuring services with account WIN-ST17FJ32SOG\ddagentuser +CA: 02:11:36: GetPreviousAgentUser. Could not find previous agent user: System.Exception: Agent user information is not in registry + at Datadog.CustomActions.InstallStateCustomActions.GetPreviousAgentUser(ISession session, IRegistryServices registryServices, INativeMethods nativeMethods) +CA: 02:11:36: UpdateAndLogAccessControl. datadog-process-agent current ACLs: D:(A;;CCLCSWLOCRRC;;;IU)(A;;CCLCSWLOCRRC;;;SU)(A;;CCLCSWRPWPDTLOCRRC;;;SY)(A;;CCDCLCSWRPWPDTLOCRSDRCWDWO;;;BA) + +--- 24843:25383 +CA(ddprocmon): DriverInstall: Done with create() 0 +CA(ddprocmon): DriverInstall: done installing services +SFXCA: Extracting custom action to temporary directory: C:\Windows\Installer\MSI593F.tmp-\ +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.StartDDServices +CustomAction WixFailWhenDeferred returned actual error code 1603 (note this may not be 100% accurate if translation happened inside sandbox) +Action ended 2:11:49: InstallFinalize. Return value 3. + +`, + }, + "Missing password for DC": { + input: "missing_password_for_dc.log", + expected: `--- 3625:5242 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.ProcessDdAgentUserCredentials +CA: 01:49:43: LookupAccountWithExtendedDomainSyntax. User not found, trying again with fixed domain part: \toto +CA: 01:49:43: ProcessDdAgentUserCredentials. User toto doesn't exist. +CA: 01:49:43: ProcessDdAgentUserCredentials. domain part is empty, using default +CA: 01:49:43: ProcessDdAgentUserCredentials. Installing with DDAGENTUSER_PROCESSED_NAME=toto and DDAGENTUSER_PROCESSED_DOMAIN=datadoghq-qa-labs.local +CA: 01:49:43: HandleProcessDdAgentUserCredentialsException. Error processing ddAgentUser credentials: Datadog.CustomActions.InvalidAgentUserConfigurationException: A password was not provided. A password is a required when installing on Domain Controllers. + at Datadog.CustomActions.ProcessUserCustomActions.ProcessDdAgentUserCredentials(Boolean calledFromUIControl) +MSI (s) (C8!50) [01:49:43:906]: Product: Datadog Agent -- A password was not provided. A password is a required when installing on Domain Controllers. + +A password was not provided. A password is a required when installing on Domain Controllers. +CustomAction ProcessDdAgentUserCredentials returned actual error code 1603 (note this may not be 100% accurate if translation happened inside sandbox) +Action ended 1:49:43: ProcessDdAgentUserCredentials. Return value 3. +Action ended 1:49:43: INSTALL. Return value 3. +Property(S): UpgradeCode = {0C50421B-AEFB-4F15-A809-7AF256D608A5} +Property(S): NETFRAMEWORK45 = #528449 +Property(S): WIXUI_EXITDIALOGOPTIONALCHECKBOX = 1 + +`, + }, + "File in use": { + input: "file_in_use.log", + expected: `--- 3557:3890 +Action start 1:45:18: InstallValidate. +Info 1603. The file C:\Program Files\Datadog\Datadog Agent\bin\agent\process-agent.exe is being held in use by the following process: Name: process-agent, Id: 4704, Window Title: '(not determined yet)'. Close that application and retry. +Action ended 1:45:21: InstallValidate. Return value 1. + +--- 5653:5849 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action CustomActions!Datadog.CustomActions.PrerequisitesCustomActions.EnsureAdminCaller +Action ended 1:45:22: RunAsAdmin. Return value 1. + +--- 5983:6218 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action CustomActions!Datadog.CustomActions.InstallStateCustomActions.ReadInstallState +CA: 01:45:23: RegistryProperty. Found DDAGENTUSER_NAME in registry DDOG-HQ-QA-LABS\ddGmsa$ + +--- 7338:7520 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action CustomActions!Datadog.CustomActions.ConfigCustomActions.ReadConfig +Action ended 1:45:25: ReadConfig. Return value 1. + +--- 7627:7960 +Action start 1:45:25: InstallValidate. +Info 1603. The file C:\Program Files\Datadog\Datadog Agent\bin\agent\process-agent.exe is being held in use by the following process: Name: process-agent, Id: 4704, Window Title: '(not determined yet)'. Close that application and retry. +Action ended 1:45:27: InstallValidate. Return value 1. + +--- 12929:13180 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action CustomActions!Datadog.CustomActions.Rollback.RestoreDaclRollbackCustomAction.DoRollback +CA: 01:45:35: DoRollback. Resetting inheritance flag on "C:\Program Files\Datadog\Datadog Agent\" + +--- 13273:13484 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action CustomActions!Datadog.CustomActions.ServiceCustomAction.StopDDServices +CA: 01:45:39: StopDDServices. Service datadog-system-probe status: Stopped + +--- 14824:15093 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action CustomActions!Datadog.CustomActions.ConfigureUserCustomActions.UninstallUser +CA: 01:45:42: UninstallUser. Removing file access for DDOG-HQ-QA-LABS\ddGmsa$ (S-1-5-21-3647231507-2031390810-2876811253-1605) + +--- 464565:465186 +Action start 1:46:16: PrepareDecompressPythonDistributions. +SFXCA: Extracting custom action to temporary directory: C:\Windows\Installer\MSI58A1.tmp-\ +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.PrepareDecompressPythonDistributions +CA: 01:46:17: PrepareDecompressPythonDistributions. Could not set the progress bar size +CustomAction PrepareDecompressPythonDistributions returned actual error code 1603 but will be translated to success due to continue marking +Action ended 1:46:17: PrepareDecompressPythonDistributions. Return value 1. + +`, + }, + "Invalid credentials": { + input: "invalid_credentials.log", + expected: `--- 7263:7883 +Action start 1:50:19: PrepareDecompressPythonDistributions. +SFXCA: Extracting custom action to temporary directory: C:\Windows\Installer\MSID56.tmp-\ +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.PrepareDecompressPythonDistributions +CA: 01:50:19: PrepareDecompressPythonDistributions. Could not set the progress bar size +CustomAction PrepareDecompressPythonDistributions returned actual error code 1603 but will be translated to success due to continue marking +Action ended 1:50:19: PrepareDecompressPythonDistributions. Return value 1. + +--- 25259:26729 +CA: 01:50:48: StoreAgentUserInRegistry. Storing installedUser=ddagentuser +CA(ddnpm): DriverInstall: Initialized +CA(ddnpm): DriverInstall: Installing services +CA(ddnpm): DriverInstall: installing service +CA(ddnpm): DriverInstall: serviceDef::create() +CA(ddnpm): DriverInstall: Failed to CreateService 1073 +CA(ddnpm): DriverInstall: Service exists, verifying +CA(ddnpm): DriverInstall: Updated path for existing service +CA(ddnpm): DriverInstall: done installing services +CA(ddapm): DriverInstall: Initialized +CA(ddapm): DriverInstall: Installing services +CA(ddapm): DriverInstall: installing service +CA(ddapm): DriverInstall: serviceDef::create() +CA(ddapm): DriverInstall: Failed to CreateService 1073 +CA(ddapm): DriverInstall: Service exists, verifying +CA(ddapm): DriverInstall: Updated path for existing service +CA(ddapm): DriverInstall: done installing services +CA(ddprocmon): DriverInstall: Initialized +CA(ddprocmon): DriverInstall: Installing services +CA(ddprocmon): DriverInstall: installing service +CA(ddprocmon): DriverInstall: serviceDef::create() +CA(ddprocmon): DriverInstall: Failed to CreateService 1073 +CA(ddprocmon): DriverInstall: Service exists, verifying +CA(ddprocmon): DriverInstall: Updated path for existing service +CA(ddprocmon): DriverInstall: done installing services +SFXCA: Extracting custom action to temporary directory: C:\Windows\Installer\MSI7FD3.tmp-\ +SFXCA: Binding to CLR version v4.0.30319 + +--- 26730:27404 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.StartDDServices +CA: 01:50:49: StartDDServices. Failed to start services: System.InvalidOperationException: Cannot start service datadogagent on computer '.'. ---> System.ComponentModel.Win32Exception: The service did not start due to a logon failure + --- End of inner exception stack trace --- + at System.ServiceProcess.ServiceController.Start(String args) + at Datadog.CustomActions.Native.ServiceController.StartService(String serviceName, TimeSpan timeout) + at Datadog.CustomActions.ServiceCustomAction.StartDDServices() +Action ended 1:50:49: InstallFinalize. Return value 1. + +`, + }, + "Service marked for deletion": { + input: "service_marked_for_deletion.log", + expected: `--- 1546:2037 +Action ended 6:11:36: LaunchConditions. Return value 1. +Action start 6:11:36: ValidateProductID. +Action ended 6:11:36: ValidateProductID. Return value 1. +Action start 6:11:36: WixRemoveFoldersEx. +WixRemoveFoldersEx: Error 0x80070057: Missing folder property: dd_PROJECTLOCATION_0 for row: RemoveFolderEx +CustomAction WixRemoveFoldersEx returned actual error code 1603 but will be translated to success due to continue marking +Action ended 6:11:36: WixRemoveFoldersEx. Return value 1. + +--- 6876:7497 +Action start 6:11:38: PrepareDecompressPythonDistributions. +SFXCA: Extracting custom action to temporary directory: C:\Windows\Installer\MSI7CB7.tmp-\ +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.PrepareDecompressPythonDistributions +CA: 06:11:39: PrepareDecompressPythonDistributions. Could not set the progress bar size +CustomAction PrepareDecompressPythonDistributions returned actual error code 1603 but will be translated to success due to continue marking +Action ended 6:11:39: PrepareDecompressPythonDistributions. Return value 1. + +--- 16762:17181 +CA: 06:12:16: AddUser. ddagentuser already exists, not creating +CA: 06:12:16: GetPreviousAgentUser. Could not find previous agent user: System.Exception: Agent user information is not in registry + at Datadog.CustomActions.InstallStateCustomActions.GetPreviousAgentUser(ISession session, IRegistryServices registryServices, INativeMethods nativeMethods) +CA: 06:12:16: ConfigureUser. Resetting ddagentuser password. + +--- 19832:20285 + } +] +MSI (s) (B4:24) [06:12:21:764]: Product: Datadog Agent -- Error 1923. Service 'Datadog Agent' (datadogagent) could not be installed. Verify that you have sufficient privileges to install system services. + +Error 1923. Service 'Datadog Agent' (datadogagent) could not be installed. Verify that you have sufficient privileges to install system services. +SFXCA: Extracting custom action to temporary directory: C:\Windows\Installer\MSI2787.tmp-\ + +--- 20286:20851 +SFXCA: Binding to CLR version v4.0.30319 +Calling custom action AgentCustomActions!Datadog.AgentCustomActions.CustomActions.EnsureNpmServiceDependency +ExecServiceConfig: Error 0x80070430: Cannot change service configuration. Error: The specified service has been marked for deletion. + +ExecServiceConfig: Error 0x80070430: Failed to configure service: datadogagent +CustomAction ExecServiceConfig returned actual error code 1603 (note this may not be 100% accurate if translation happened inside sandbox) +Action ended 6:12:22: InstallFinalize. Return value 3. + +`, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + file, err := logFilesFS.Open(test.input) + require.NoError(t, err) + got, err := m.processLogFile(file) + require.NoError(t, err) + resultString := strings.ReplaceAll(string(got), "\r", "") + require.Equal(t, test.expected, resultString) + }) + } +} diff --git a/pkg/fleet/installer/packages/msiexec.go b/pkg/fleet/internal/msi/product.go similarity index 59% rename from pkg/fleet/installer/packages/msiexec.go rename to pkg/fleet/internal/msi/product.go index e65692bb38c8f..e87144d748f9d 100644 --- a/pkg/fleet/installer/packages/msiexec.go +++ b/pkg/fleet/internal/msi/product.go @@ -5,34 +5,14 @@ //go:build windows -package packages +package msi import ( "fmt" - "os/exec" - "path/filepath" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" - "github.com/DataDog/datadog-agent/pkg/util/log" "golang.org/x/sys/windows/registry" ) -func msiexec(target, product, operation string, args []string) (*exec.Cmd, error) { - updaterPath := filepath.Join(paths.PackagesPath, product, target) - msis, err := filepath.Glob(filepath.Join(updaterPath, fmt.Sprintf("%s-*-1-x86_64.msi", product))) - if err != nil { - return nil, err - } - if len(msis) > 1 { - return nil, fmt.Errorf("too many MSIs in package") - } else if len(msis) == 0 { - return nil, fmt.Errorf("no MSIs in package") - } - - cmd := exec.Command("msiexec", append([]string{operation, msis[0], "/qn", "MSIFASTINSTALL=7"}, args...)...) - return cmd, nil -} - // Product represents a software from the Windows Registry type Product struct { // Code is the software product code @@ -41,7 +21,8 @@ type Product struct { UninstallString string } -func findProductCode(name string) (*Product, error) { +// FindProductCode looks for the productName in the registry and returns information about it +func FindProductCode(productName string) (*Product, error) { rootPath := "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall" reg, err := registry.OpenKey(registry.LOCAL_MACHINE, rootPath, registry.ENUMERATE_SUB_KEYS) if err != nil { @@ -53,55 +34,55 @@ func findProductCode(name string) (*Product, error) { return nil, err } for _, key := range keys { - product, err := processKey(rootPath, key, name) + product, err := processKey(rootPath, key, productName) if err == nil && product != nil { - return product, err + return product, nil } } - return nil, nil + return nil, fmt.Errorf("product not found") } -func processKey(rootPath, key, name string) (*Product, error) { - subkey, err := registry.OpenKey(registry.LOCAL_MACHINE, rootPath+"\\"+key, registry.QUERY_VALUE) +// IsProductInstalled returns true if the given productName is installed +func IsProductInstalled(productName string) bool { + product, err := FindProductCode(productName) if err != nil { - return nil, err - } - defer subkey.Close() - - displayName, _, err := subkey.GetStringValue("DisplayName") - if err == nil && displayName == name { - product := &Product{} - product.UninstallString, _, _ = subkey.GetStringValue("UninstallString") - product.Code = key - return product, nil + return false } - - return nil, nil + return product != nil } -// removeProduct uses the registry to try and find a product and use msiexec to remove it. +// RemoveProduct uses the registry to try and find a product and use msiexec to remove it. // It is different from msiexec in that it uses the registry and not the stable/experiment path on disk to // uninstall the product. // This is needed because in certain circumstances the installer database stored in the stable/experiment paths does not // reflect the installed version, and using those installers can lead to undefined behavior (either failure to uninstall, // or weird bugs from uninstalling a product with an installer from a different version). -func removeProduct(productName string) error { - log.Debugf("Removing product %s", productName) - product, err := findProductCode(productName) +func RemoveProduct(productName string) error { + cmd, err := Cmd(Uninstall(), WithProduct(productName)) if err != nil { - return fmt.Errorf("error trying to find product %s: %w", productName, err) + return fmt.Errorf("failed to remove product: %w", err) } - if product != nil { - cmd := exec.Command("msiexec", "/x", product.Code, "/qn", "MSIFASTINSTALL=7") - return cmd.Run() + output, err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to remove product: %w\n%s", err, string(output)) } - return fmt.Errorf("product %s not found", productName) + return nil } -func isProductInstalled(productName string) bool { - product, err := findProductCode(productName) +func processKey(rootPath, key, name string) (*Product, error) { + subkey, err := registry.OpenKey(registry.LOCAL_MACHINE, rootPath+"\\"+key, registry.QUERY_VALUE) if err != nil { - return false + return nil, err } - return product != nil + defer subkey.Close() + + displayName, _, err := subkey.GetStringValue("DisplayName") + if err == nil && displayName == name { + product := &Product{} + product.UninstallString, _, _ = subkey.GetStringValue("UninstallString") + product.Code = key + return product, nil + } + + return nil, nil } diff --git a/pkg/fleet/internal/msi/service_marked_for_deletion.log b/pkg/fleet/internal/msi/service_marked_for_deletion.log new file mode 100644 index 0000000000000..7844b7a546931 Binary files /dev/null and b/pkg/fleet/internal/msi/service_marked_for_deletion.log differ diff --git a/pkg/fleet/internal/msi/wixfailwhendeferred.log b/pkg/fleet/internal/msi/wixfailwhendeferred.log new file mode 100644 index 0000000000000..fb1982434be9b Binary files /dev/null and b/pkg/fleet/internal/msi/wixfailwhendeferred.log differ diff --git a/pkg/fleet/telemetry/http_wrapper.go b/pkg/fleet/telemetry/http_wrapper.go new file mode 100644 index 0000000000000..2f94bda79fe1f --- /dev/null +++ b/pkg/fleet/telemetry/http_wrapper.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "net/http" + "strconv" + "strings" +) + +// WrapRoundTripper wraps the round tripper with the telemetry round tripper. +func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { + if rt == nil { + rt = http.DefaultTransport + } + if wrapped, ok := rt.(*roundTripper); ok { + rt = wrapped.base + } + return &roundTripper{ + base: rt, + } +} + +type roundTripper struct { + base http.RoundTripper +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + span, _ := StartSpanFromContext(req.Context(), "http.request") + defer func() { span.Finish(err) }() + + url := *req.URL + url.User = nil + + span.span.Type = "http" + span.SetResourceName(req.Method + " " + urlFromRequest(req)) + span.span.Meta["http.method"] = req.Method + span.span.Meta["http.url"] = req.URL.String() + span.span.Meta["span.kind"] = "client" + span.span.Meta["network.destination.name"] = url.Hostname() + res, err = rt.base.RoundTrip(req) + if err != nil { + span.SetTag("http.errors", err.Error()) + return res, err + } + span.SetTag("http.status_code", strconv.Itoa(res.StatusCode)) + if res.StatusCode >= 400 { + span.SetTag("http.errors", res.Status) + } + return res, err +} + +// urlFromRequest returns the URL from the HTTP request. The URL query string is included in the return object iff queryString is true +// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information. +func urlFromRequest(r *http.Request) string { + // Quoting net/http comments about net.Request.URL on server requests: + // "For most requests, fields other than Path and RawQuery will be + // empty. (See RFC 7230, Section 5.3)" + // This is why we don't rely on url.URL.String(), url.URL.Host, url.URL.Scheme, etc... + var url string + path := r.URL.EscapedPath() + scheme := r.URL.Scheme + if r.TLS != nil { + scheme = "https" + } + if r.Host != "" { + url = strings.Join([]string{scheme, "://", r.Host, path}, "") + } else { + url = path + } + // Collect the query string if we are allowed to report it and obfuscate it if possible/allowed + if r.URL.RawQuery != "" { + query := r.URL.RawQuery + url = strings.Join([]string{url, query}, "?") + } + if frag := r.URL.EscapedFragment(); frag != "" { + url = strings.Join([]string{url, frag}, "#") + } + return url +} diff --git a/pkg/fleet/telemetry/span.go b/pkg/fleet/telemetry/span.go index ac8f79516c736..ac5f00b9ade0a 100644 --- a/pkg/fleet/telemetry/span.go +++ b/pkg/fleet/telemetry/span.go @@ -7,20 +7,144 @@ package telemetry import ( - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "context" + "fmt" + "math/rand/v2" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" ) -// Span is an alias for ddtrace.Span until we phase ddtrace out. -type Span struct{ ddtrace.Span } +const spanKey = spanContextKey("span_context") + +type spanContextKey string + +// Span represents a span. +type Span struct { + mu sync.Mutex + span internaltelemetry.Span + finished atomic.Bool +} + +func newSpan(name string, parentID, traceID uint64) *Span { + if traceID == 0 { + traceID = rand.Uint64() + if !headSamplingKeep(name, traceID) { + traceID = dropTraceID + } + } + s := &Span{ + span: internaltelemetry.Span{ + TraceID: traceID, + ParentID: parentID, + SpanID: rand.Uint64(), + Name: name, + Resource: name, + Start: time.Now().UnixNano(), + Meta: make(map[string]string), + Metrics: make(map[string]float64), + }, + } + if parentID == 0 { + s.SetTopLevel() + } + + globalTracer.registerSpan(s) + return s +} // Finish finishes the span with an error. func (s *Span) Finish(err error) { - s.Span.Finish(tracer.WithError(err)) + s.finished.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + s.span.Duration = time.Now().UnixNano() - s.span.Start + if err != nil { + s.span.Error = 1 + s.span.Meta = map[string]string{ + "error.message": err.Error(), + "error.stack": string(debug.Stack()), + } + } + globalTracer.finishSpan(s) } // SetResourceName sets the resource name of the span. func (s *Span) SetResourceName(name string) { - s.Span.SetTag(ext.ResourceName, name) + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.span.Resource = name +} + +// SetTopLevel sets the span as a top level span. +func (s *Span) SetTopLevel() { + s.SetTag("_top_level", 1) +} + +// SetTag sets a tag on the span. +func (s *Span) SetTag(key string, value interface{}) { + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if value == nil { + s.span.Meta[key] = "nil" + } + switch v := value.(type) { + case string: + s.span.Meta[key] = v + case bool: + s.span.Meta[key] = strconv.FormatBool(v) + case int: + s.span.Metrics[key] = float64(v) + case int8: + s.span.Metrics[key] = float64(v) + case int16: + s.span.Metrics[key] = float64(v) + case int32: + s.span.Metrics[key] = float64(v) + case int64: + s.span.Metrics[key] = float64(v) + case uint: + s.span.Metrics[key] = float64(v) + case uint8: + s.span.Metrics[key] = float64(v) + case uint16: + s.span.Metrics[key] = float64(v) + case uint32: + s.span.Metrics[key] = float64(v) + case uint64: + s.span.Metrics[key] = float64(v) + case float32: + s.span.Metrics[key] = float64(v) + case float64: + s.span.Metrics[key] = v + default: + s.span.Meta[key] = fmt.Sprintf("not_supported_type %T", v) + } +} + +type spanIDs struct { + traceID uint64 + spanID uint64 +} + +func getSpanIDsFromContext(ctx context.Context) (spanIDs, bool) { + sIDs, ok := ctx.Value(spanKey).(spanIDs) + if !ok { + return spanIDs{}, false + } + return sIDs, true +} + +func setSpanIDsInContext(ctx context.Context, span *Span) context.Context { + return context.WithValue(ctx, spanKey, spanIDs{traceID: span.span.TraceID, spanID: span.span.SpanID}) } diff --git a/pkg/fleet/telemetry/telemetry.go b/pkg/fleet/telemetry/telemetry.go index 5bbfd8ca773c4..66174ad18fc32 100644 --- a/pkg/fleet/telemetry/telemetry.go +++ b/pkg/fleet/telemetry/telemetry.go @@ -8,280 +8,178 @@ package telemetry import ( "context" - "errors" "fmt" - "io" - "math/rand/v2" - "net" "net/http" "os" "strconv" "strings" - "sync" - - httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - - "github.com/gorilla/mux" + "time" "github.com/DataDog/datadog-agent/pkg/internaltelemetry" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - traceconfig "github.com/DataDog/datadog-agent/pkg/trace/config" - "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" ) const ( - // EnvTraceID is the environment variable key for the trace ID - EnvTraceID = "DATADOG_TRACE_ID" - // EnvParentID is the environment variable key for the parent ID - EnvParentID = "DATADOG_PARENT_ID" -) - -const ( + envTraceID = "DATADOG_TRACE_ID" + envParentID = "DATADOG_PARENT_ID" telemetrySubdomain = "instrumentation-telemetry-intake" - telemetryEndpoint = "/v0.4/traces" ) // Telemetry handles the telemetry for fleet components. type Telemetry struct { telemetryClient internaltelemetry.Client + done chan struct{} + flushed chan struct{} - site string + env string service string - - listener *telemetryListener - server *http.Server - client *http.Client - - samplingRules []tracer.SamplingRule } -// Option is a functional option for telemetry. -type Option func(*Telemetry) - // NewTelemetry creates a new telemetry instance -func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) *Telemetry { - endpoint := &traceconfig.Endpoint{ +func NewTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + t := newTelemetry(client, apiKey, site, service) + t.Start() + return t +} + +func newTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + endpoint := &internaltelemetry.Endpoint{ Host: fmt.Sprintf("https://%s.%s", telemetrySubdomain, strings.TrimSpace(site)), APIKey: apiKey, } - listener := newTelemetryListener() - t := &Telemetry{ - telemetryClient: internaltelemetry.NewClient(client, []*traceconfig.Endpoint{endpoint}, service, site == "datad0g.com"), - site: site, - service: service, - listener: listener, - server: &http.Server{}, - client: &http.Client{ - Transport: &http.Transport{ - Dial: listener.Dial, - }, - }, + env := "prod" + if site == "datad0g.com" { + env = "staging" } - for _, opt := range opts { - opt(t) + + return &Telemetry{ + telemetryClient: internaltelemetry.NewClient(client, []*internaltelemetry.Endpoint{endpoint}, service, site == "datad0g.com"), + done: make(chan struct{}), + flushed: make(chan struct{}), + env: env, + service: service, } - t.server.Handler = t.handler() - return t } // Start starts the telemetry -func (t *Telemetry) Start(_ context.Context) error { +func (t *Telemetry) Start() { + ticker := time.Tick(1 * time.Minute) go func() { - err := t.server.Serve(t.listener) - if err != nil { - log.Infof("telemetry server stopped: %v", err) + for { + select { + case <-ticker: + t.sendCompletedSpans() + case <-t.done: + t.sendCompletedSpans() + close(t.flushed) + return + } } }() - env := "prod" - if t.site == "datad0g.com" { - env = "staging" - } - - tracer.Start( - tracer.WithService(t.service), - tracer.WithServiceVersion(version.AgentVersion), - tracer.WithEnv(env), - tracer.WithGlobalTag("site", t.site), - tracer.WithHTTPClient(t.client), - tracer.WithLogStartup(false), - - // We don't need the value, we just need to enforce that it's not - // the default. If it is, then the tracer will try to use the socket - // if it exists -- and it always exists for newer agents. - // If the agent address is the socket, the tracer overrides WithHTTPClient to use it. - tracer.WithAgentAddr("192.0.2.42:12345"), // 192.0.2.0/24 is reserved - tracer.WithSamplingRules(t.samplingRules), - ) - return nil } // Stop stops the telemetry -func (t *Telemetry) Stop(ctx context.Context) error { - tracer.Flush() - tracer.Stop() - t.listener.Close() - err := t.server.Shutdown(ctx) - if err != nil { - log.Errorf("error shutting down telemetry server: %v", err) - } - return nil -} - -func (t *Telemetry) handler() http.Handler { - r := mux.NewRouter().Headers("Content-Type", "application/msgpack").Subrouter() - r.HandleFunc(telemetryEndpoint, func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - log.Errorf("error reading request body: %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - var traces pb.Traces - _, err = traces.UnmarshalMsg(body) - if err != nil { - log.Errorf("error unmarshalling traces: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - t.telemetryClient.SendTraces(traces) - w.WriteHeader(http.StatusOK) - }) - return r +func (t *Telemetry) Stop() { + close(t.done) + <-t.flushed } -type telemetryListener struct { - conns chan net.Conn - - close chan struct{} - closeOnce sync.Once -} - -func newTelemetryListener() *telemetryListener { - return &telemetryListener{ - conns: make(chan net.Conn), - close: make(chan struct{}), +func (t *Telemetry) extractCompletedSpans() internaltelemetry.Traces { + spans := globalTracer.flushCompletedSpans() + if len(spans) == 0 { + return internaltelemetry.Traces{} } -} - -func (l *telemetryListener) Close() error { - l.closeOnce.Do(func() { - close(l.close) - }) - return nil -} - -func (l *telemetryListener) Accept() (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - case conn := <-l.conns: - return conn, nil + traces := make(map[uint64][]*internaltelemetry.Span) + for _, span := range spans { + span.span.Service = t.service + span.span.Meta["env"] = t.env + span.span.Meta["version"] = version.AgentVersion + span.span.Metrics["_sampling_priority_v1"] = 2 + traces[span.span.TraceID] = append(traces[span.span.TraceID], &span.span) + } + tracesArray := make([]internaltelemetry.Trace, 0, len(traces)) + for _, trace := range traces { + tracesArray = append(tracesArray, internaltelemetry.Trace(trace)) } + return internaltelemetry.Traces(tracesArray) } -func (l *telemetryListener) Addr() net.Addr { - return addr(0) +func (t *Telemetry) sendCompletedSpans() { + tracesArray := t.extractCompletedSpans() + if len(tracesArray) == 0 { + return + } + t.telemetryClient.SendTraces(tracesArray) } -func (l *telemetryListener) Dial(_, _ string) (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - default: +// SpanFromContext returns the span from the context if available. +func SpanFromContext(ctx context.Context) (*Span, bool) { + spanIDs, ok := getSpanIDsFromContext(ctx) + if !ok { + return nil, false } - server, client := net.Pipe() - l.conns <- server - return client, nil + return globalTracer.getSpan(spanIDs.spanID) } -type addr int - -func (addr) Network() string { - return "memory" +// StartSpanFromEnv starts a span using the environment variables to find the parent span. +func StartSpanFromEnv(ctx context.Context, operationName string) (*Span, context.Context) { + traceID, parentID := extractIDsFromEnv() + return StartSpanFromIDs(ctx, operationName, traceID, parentID) } -func (addr) String() string { - return "local" +func extractIDsFromEnv() (string, string) { + parentID, ok := os.LookupEnv(envParentID) + if !ok { + return "0", "0" + } + traceID, ok := os.LookupEnv(envTraceID) + if !ok { + return "0", "0" + } + return traceID, parentID } -// StartSpanFromIDs starts a span using the trace and parent -// IDs provided. -func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - ctxCarrier := tracer.TextMapCarrier{ - tracer.DefaultTraceIDHeader: traceID, - tracer.DefaultParentIDHeader: parentID, - tracer.DefaultPriorityHeader: "2", +func converIDsToUint64(traceID, parentID string) (uint64, uint64) { + traceIDInt, err := strconv.ParseUint(traceID, 10, 64) + if err != nil { + return 0, 0 } - spanCtx, err := tracer.Extract(ctxCarrier) + parentIDInt, err := strconv.ParseUint(parentID, 10, 64) if err != nil { - log.Debugf("failed to extract span context from install script params: %v", err) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return 0, 0 } - spanOptions = append([]ddtrace.StartSpanOption{tracer.ChildOf(spanCtx)}, spanOptions...) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return traceIDInt, parentIDInt } -// SpanFromContext returns the span from the context if available. -func SpanFromContext(ctx context.Context) (Span, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return Span{}, false - } - return Span{span}, true +// StartSpanFromIDs starts a span using the trace and parent +// IDs provided. +func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string) (*Span, context.Context) { + traceIDInt, parentIDInt := converIDsToUint64(traceID, parentID) + span, ctx := startSpanFromIDs(ctx, operationName, traceIDInt, parentIDInt) + span.SetTopLevel() + return span, ctx } -// StartSpanFromContext starts a span using the context to find the parent span. -func StartSpanFromContext(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - span, ctx := tracer.StartSpanFromContext(ctx, operationName, spanOptions...) - return Span{span}, ctx +func startSpanFromIDs(ctx context.Context, operationName string, traceID, parentID uint64) (*Span, context.Context) { + s := newSpan(operationName, parentID, traceID) + ctx = setSpanIDsInContext(ctx, s) + return s, ctx } -// StartSpanFromEnv starts a span using the environment variables to find the parent span. -func StartSpanFromEnv(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - traceID, ok := os.LookupEnv(EnvTraceID) - if !ok { - traceID = strconv.FormatUint(rand.Uint64(), 10) - } - parentID, ok := os.LookupEnv(EnvParentID) - if !ok { - parentID = "0" - } - return StartSpanFromIDs(ctx, operationName, traceID, parentID, spanOptions...) +// StartSpanFromContext starts a span using the context to find the parent span. +func StartSpanFromContext(ctx context.Context, operationName string) (*Span, context.Context) { + spanIDs, _ := getSpanIDsFromContext(ctx) + return startSpanFromIDs(ctx, operationName, spanIDs.traceID, spanIDs.spanID) } // EnvFromContext returns the environment variables for the context. func EnvFromContext(ctx context.Context) []string { - spanCtx, ok := SpanContextFromContext(ctx) + sIDs, ok := getSpanIDsFromContext(ctx) if !ok { return []string{} } return []string{ - fmt.Sprintf("%s=%d", EnvTraceID, spanCtx.TraceID()), - fmt.Sprintf("%s=%d", EnvParentID, spanCtx.SpanID()), + fmt.Sprintf("%s=%s", envTraceID, strconv.FormatUint(sIDs.traceID, 10)), + fmt.Sprintf("%s=%s", envParentID, strconv.FormatUint(sIDs.spanID, 10)), } } - -// SpanContextFromContext extracts the span context from the context if available. -func SpanContextFromContext(ctx context.Context) (ddtrace.SpanContext, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return nil, false - } - return span.Context(), true -} - -// WithSamplingRules sets the sampling rules for the telemetry. -func WithSamplingRules(rules ...tracer.SamplingRule) Option { - return func(t *Telemetry) { - t.samplingRules = rules - } -} - -// WrapRoundTripper wraps the round tripper with the telemetry round tripper. -func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { - return httptrace.WrapRoundTripper(rt) -} diff --git a/pkg/fleet/telemetry/telemetry_test.go b/pkg/fleet/telemetry/telemetry_test.go new file mode 100644 index 0000000000000..d0866ee4d4a13 --- /dev/null +++ b/pkg/fleet/telemetry/telemetry_test.go @@ -0,0 +1,222 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "context" + "errors" + "net/http" + "os" + "testing" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFreshSpan(t *testing.T) { + ctx := context.Background() + _, ok := SpanFromContext(ctx) + require.False(t, ok) + + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + s.SetResourceName("new") + + span, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, s, span) + + assert.Equal(t, "test", s.span.Name) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", span.span.Resource) +} + +func TestInheritence(t *testing.T) { + ctx := context.Background() + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + + child, _ := StartSpanFromContext(ctx, "child") + require.NotNil(t, child) + + assert.Equal(t, s.span.SpanID, child.span.ParentID) + assert.Equal(t, s.span.TraceID, child.span.TraceID) +} + +func TestStartSpanFromIDs(t *testing.T) { + ctx := context.Background() + traceID := "100" + parentID := "200" + + span, ctx := StartSpanFromIDs(ctx, "ids-operation", traceID, parentID) + require.NotNil(t, span, "Expected a span") + require.Equal(t, uint64(100), span.span.TraceID) + require.Equal(t, uint64(200), span.span.ParentID) + + val, ok := span.span.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + + spanFromCtx, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, span, spanFromCtx) +} + +func strPtr(s string) *string { + return &s +} + +func TestSpanFromEnv(t *testing.T) { + randTraceID := uint64(9) + tt := []struct { + name string + envTraceID *string + envParentID *string + expectedTraceID uint64 + expectedParentID uint64 + }{ + { + name: "no parent env", + envTraceID: strPtr("100"), + envParentID: nil, + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "no trace env", + envTraceID: nil, + envParentID: strPtr("100"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "traceID malformed", + envTraceID: strPtr("not-a-number"), + envParentID: strPtr("200"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "parentID malformed", + envTraceID: strPtr("100"), + envParentID: strPtr("not-a-number"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "inheritance", + envTraceID: strPtr("100"), + envParentID: strPtr("200"), + expectedTraceID: 100, + expectedParentID: 200, + }, + } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + if tc.envTraceID != nil { + os.Setenv(envTraceID, *tc.envTraceID) + defer os.Unsetenv(envTraceID) + } + if tc.envParentID != nil { + os.Setenv(envParentID, *tc.envParentID) + defer os.Unsetenv(envParentID) + } + + span, ctx := StartSpanFromEnv(context.Background(), "env-operation") + require.NotNil(t, span, "Expected a span") + s, ok := SpanFromContext(ctx) + assert.True(t, ok) + assert.Equal(t, span, s) + + assert.Equal(t, tc.expectedParentID, span.span.ParentID) + if tc.expectedTraceID != randTraceID { + assert.Equal(t, tc.expectedTraceID, span.span.TraceID) + } else { + assert.NotEqual(t, 0, span.span.TraceID) + } + + }) + } +} + +func TestLimit(t *testing.T) { + totalSpans := maxSpansInFlight + 2 + ctx := context.Background() + for i := 0; i < totalSpans; i++ { + _, ctx = StartSpanFromContext(ctx, "test") + } + assert.Len(t, globalTracer.spans, maxSpansInFlight) +} + +func TestEnvFromContext(t *testing.T) { + s, ctx := StartSpanFromContext(context.Background(), "test") + s.span.TraceID = 456 + s.span.SpanID = 123 + ctx = setSpanIDsInContext(ctx, s) + env := EnvFromContext(ctx) + assert.ElementsMatch(t, []string{"DATADOG_TRACE_ID=456", "DATADOG_PARENT_ID=123"}, env) + + env = EnvFromContext(context.Background()) + assert.ElementsMatch(t, []string{}, env) +} + +func TestSpanFinished(t *testing.T) { + s, _ := StartSpanFromContext(context.Background(), "test") + s.Finish(nil) + s.SetResourceName("new") + s.SetTag("key", "value") + + assert.Equal(t, "test", s.span.Resource) + _, ok := s.span.Meta["key"] + assert.False(t, ok) +} + +func TestRemapOnFlush(t *testing.T) { + const testService = "test-service" + const numTraces = 10 + telem := newTelemetry(&http.Client{}, "api", "datad0g.com", testService) + globalTracer = &tracer{spans: make(map[uint64]*Span)} + + // traces with 2 spans + for i := 0; i < numTraces; i++ { + parentSpan, ctx := StartSpanFromContext(context.Background(), "parent") + childSpan, _ := StartSpanFromContext(ctx, "child") + childSpan.Finish(errors.New("test_error")) + parentSpan.Finish(nil) + } + resTraces := telem.extractCompletedSpans() + require.Len(t, resTraces, numTraces) + + for _, trace := range resTraces { + assert.Len(t, trace, 2) + for _, span := range trace { + assert.Equal(t, testService, span.Service) + assert.Equal(t, "staging", span.Meta["env"]) + assert.Equal(t, 2.0, span.Metrics["_sampling_priority_v1"]) + } + var parent, child *internaltelemetry.Span + if trace[0].Name == "parent" { + parent = trace[0] + child = trace[1] + } else { + parent = trace[1] + child = trace[0] + } + assert.Equal(t, parent.SpanID, child.ParentID) + val, ok := parent.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + _, ok = child.Metrics["_top_level"] + require.False(t, ok) + + require.Equal(t, int32(1), child.Error) + require.Equal(t, "test_error", child.Meta["error.message"]) + require.Contains(t, child.Meta["error.stack"], "telemetry_test.go") + } +} diff --git a/pkg/fleet/telemetry/tracer.go b/pkg/fleet/telemetry/tracer.go new file mode 100644 index 0000000000000..0923b3563e457 --- /dev/null +++ b/pkg/fleet/telemetry/tracer.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "math" + "strings" + "sync" +) + +const ( + dropTraceID = 1 + maxSpansInFlight = 1000 +) + +var ( + globalTracer *tracer + samplingRates = map[string]float64{ + "cdn": 0.1, + "garbage_collect": 0.05, + "HTTPClient": 0.05, + } +) + +func init() { + globalTracer = &tracer{ + spans: make(map[uint64]*Span), + } +} + +type tracer struct { + mu sync.Mutex + spans map[uint64]*Span + completedSpans []*Span +} + +func (t *tracer) registerSpan(span *Span) { + if span.span.TraceID == dropTraceID { + return + } + t.mu.Lock() + defer t.mu.Unlock() + // naive maxSpansInFlight check as this is just telemetry + // next iteration if needed would be to flush long running spans to troubleshoot + if len(t.spans) >= maxSpansInFlight { + return + } + t.spans[span.span.SpanID] = span +} + +func (t *tracer) getSpan(spanID uint64) (*Span, bool) { + t.mu.Lock() + defer t.mu.Unlock() + span, ok := t.spans[spanID] + return span, ok +} + +func (t *tracer) finishSpan(span *Span) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.spans, span.span.SpanID) + t.completedSpans = append(t.completedSpans, span) +} + +func (t *tracer) flushCompletedSpans() []*Span { + t.mu.Lock() + defer t.mu.Unlock() + newSpanArray := make([]*Span, 0) + completedSpans := t.completedSpans + t.completedSpans = newSpanArray + return completedSpans +} + +func headSamplingKeep(spanName string, traceID uint64) bool { + for k, r := range samplingRates { + if strings.Contains(spanName, k) { + return sampledByRate(traceID, r) + } + } + return true +} + +func sampledByRate(n uint64, rate float64) bool { + if rate < 1 { + return n*uint64(1111111111111111111) < uint64(rate*math.MaxUint64) + } + return true +} diff --git a/pkg/gohai/go.mod b/pkg/gohai/go.mod index 45edc82c08f38..e7dee142ec26f 100644 --- a/pkg/gohai/go.mod +++ b/pkg/gohai/go.mod @@ -26,7 +26,6 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/gohai/go.sum b/pkg/gohai/go.sum index 99c1b368ae068..828633a9495c7 100644 --- a/pkg/gohai/go.sum +++ b/pkg/gohai/go.sum @@ -44,7 +44,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/gpu/consumer.go b/pkg/gpu/consumer.go index f4ebb7f04b0f2..55148350b12e3 100644 --- a/pkg/gpu/consumer.go +++ b/pkg/gpu/consumer.go @@ -220,7 +220,7 @@ func (c *cudaEventConsumer) getStreamKey(header *gpuebpf.CudaEventHeader) stream // Try to get the GPU device if we can, but do not fail if we can't as we want to report // the data even if we can't get the GPU UUID - gpuDevice, err := c.sysCtx.getCurrentActiveGpuDevice(int(pid), int(tid)) + gpuDevice, err := c.sysCtx.getCurrentActiveGpuDevice(int(pid), int(tid), containerID) if err != nil { log.Warnf("Error getting GPU device for process %d: %v", pid, err) } else { diff --git a/pkg/gpu/consumer_test.go b/pkg/gpu/consumer_test.go index 6530f2bb49738..c7c7aa04837d7 100644 --- a/pkg/gpu/consumer_test.go +++ b/pkg/gpu/consumer_test.go @@ -22,7 +22,7 @@ import ( func TestConsumerCanStartAndStop(t *testing.T) { handler := ddebpf.NewRingBufferHandler(consumerChannelSize) cfg := config.New() - ctx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) + ctx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) consumer := newCudaEventConsumer(ctx, handler, cfg) diff --git a/pkg/gpu/context.go b/pkg/gpu/context.go index 77bac5185339f..59d778bfea5c1 100644 --- a/pkg/gpu/context.go +++ b/pkg/gpu/context.go @@ -16,10 +16,15 @@ import ( "github.com/NVIDIA/go-nvml/pkg/nvml" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/gpu/cuda" "github.com/DataDog/datadog-agent/pkg/util/ktime" + "github.com/DataDog/datadog-agent/pkg/util/log" ) +const nvidiaResourceName = "nvidia.com/gpu" + // systemContext holds certain attributes about the system that are used by the GPU probe. type systemContext struct { // maxGpuThreadsPerDevice maps each device index to the maximum number of threads it can run in parallel @@ -60,6 +65,9 @@ type systemContext struct { // visibleDevicesCache is a cache of visible devices for each process, to avoid // looking into the environment variables every time visibleDevicesCache map[int][]nvml.Device + + // workloadmeta is the workloadmeta component that we use to get necessary container metadata + workloadmeta workloadmeta.Component } // symbolsEntry embeds cuda.Symbols adding a field for keeping track of the last @@ -73,7 +81,7 @@ func (e *symbolsEntry) updateLastUsedTime() { e.lastUsedTime = time.Now() } -func getSystemContext(nvmlLib nvml.Interface, procRoot string) (*systemContext, error) { +func getSystemContext(nvmlLib nvml.Interface, procRoot string, wmeta workloadmeta.Component) (*systemContext, error) { ctx := &systemContext{ maxGpuThreadsPerDevice: make(map[int]int), deviceSmVersions: make(map[int]int), @@ -83,6 +91,7 @@ func getSystemContext(nvmlLib nvml.Interface, procRoot string) (*systemContext, procRoot: procRoot, selectedDeviceByPIDAndTID: make(map[int]map[int]int32), visibleDevicesCache: make(map[int][]nvml.Device), + workloadmeta: wmeta, } if err := ctx.fillDeviceInfo(); err != nil { @@ -209,13 +218,75 @@ func (ctx *systemContext) cleanupOldEntries() { } } +// filterDevicesForContainer filters the available GPU devices for the given +// container. If the ID is not empty, we check the assignment of GPU resources +// to the container and return only the devices that are available to the +// container. +func (ctx *systemContext) filterDevicesForContainer(devices []nvml.Device, containerID string) ([]nvml.Device, error) { + if containerID == "" { + // If the process is not running in a container, we assume all devices are available. + return devices, nil + } + + container, err := ctx.workloadmeta.GetContainer(containerID) + if err != nil { + // If we don't find the container, we assume all devices are available. + // This can happen sometimes, e.g. if we don't have the container in the + // store yet. Do not block metrics on that. + if errors.IsNotFound(err) { + return devices, nil + } + + // Some error occurred while retrieving the container, this could be a + // general error with the store, report it. + return nil, fmt.Errorf("cannot retrieve data for container %s: %s", containerID, err) + } + + var filteredDevices []nvml.Device + for _, resource := range container.AllocatedResources { + // Only consider NVIDIA GPUs + if resource.Name != nvidiaResourceName { + continue + } + + for _, device := range devices { + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + log.Warnf("Error getting GPU UUID for device %s: %s", device, nvml.ErrorString(ret)) + continue + } + + if resource.ID == uuid { + filteredDevices = append(filteredDevices, device) + break + } + } + } + + // We didn't find any devices assigned to the container, report it as an error. + if len(filteredDevices) == 0 { + return nil, fmt.Errorf("no GPU devices found for container %s that matched its allocated resources %+v", containerID, container.AllocatedResources) + } + + return filteredDevices, nil +} + // getCurrentActiveGpuDevice returns the active GPU device for a given process and thread, based on the // last selection (via cudaSetDevice) this thread made and the visible devices for the process. -func (ctx *systemContext) getCurrentActiveGpuDevice(pid int, tid int) (nvml.Device, error) { +func (ctx *systemContext) getCurrentActiveGpuDevice(pid int, tid int, containerID string) (nvml.Device, error) { visibleDevices, ok := ctx.visibleDevicesCache[pid] if !ok { - var err error - visibleDevices, err = cuda.GetVisibleDevicesForProcess(ctx.gpuDevices, pid, ctx.procRoot) + // Order is important! We need to filter the devices for the container + // first. In a container setting, the environment variable acts as a + // filter on the devices that are available to the process, not on the + // devices available on the host system. + var err error // avoid shadowing visibleDevices, declare error before so we can use = instead of := + visibleDevices, err = ctx.filterDevicesForContainer(ctx.gpuDevices, containerID) + if err != nil { + return nil, fmt.Errorf("error filtering devices for container %s: %w", containerID, err) + } + + visibleDevices, err = cuda.GetVisibleDevicesForProcess(visibleDevices, pid, ctx.procRoot) if err != nil { return nil, fmt.Errorf("error getting visible devices for process %d: %w", pid, err) } diff --git a/pkg/gpu/context_test.go b/pkg/gpu/context_test.go new file mode 100644 index 0000000000000..dbb6e1735df24 --- /dev/null +++ b/pkg/gpu/context_test.go @@ -0,0 +1,176 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux_bpf + +package gpu + +import ( + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/ebpf/uprobes" + "github.com/DataDog/datadog-agent/pkg/gpu/testutil" + "github.com/DataDog/datadog-agent/pkg/util/kernel" +) + +func TestFilterDevicesForContainer(t *testing.T) { + wmetaMock := testutil.GetWorkloadMetaMock(t) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), wmetaMock) + require.NotNil(t, sysCtx) + require.NoError(t, err) + + // Create a container with a single GPU and add it to the store + containerID := "abcdef" + deviceIndex := 2 + gpuUUID := testutil.GPUUUIDs[deviceIndex] + container := &workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindContainer, + ID: containerID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: containerID, + }, + AllocatedResources: []workloadmeta.ContainerAllocatedResource{ + { + Name: nvidiaResourceName, + ID: gpuUUID, + }, + }, + } + + wmetaMock.Set(container) + storeContainer, err := wmetaMock.GetContainer(containerID) + require.NoError(t, err, "container should be found in the store") + require.NotNil(t, storeContainer, "container should be found in the store") + + t.Run("NoContainer", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, "") + require.NoError(t, err) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices) // With no container, all devices should be returned + }) + + t.Run("NonExistentContainer", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, "non-existent-at-all") + require.NoError(t, err) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices) // If we can't find the container, all devices should be returned + }) + + t.Run("ContainerWithGPU", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, containerID) + require.NoError(t, err) + require.Len(t, filtered, 1) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices[deviceIndex:deviceIndex+1]) + }) +} + +func TestGetCurrentActiveGpuDevice(t *testing.T) { + pidNoContainer := 1234 + pidNoContainerButEnv := 2235 + pidContainer := 3238 + pidContainerAndEnv := 3239 + + envVisibleDevices := []int32{1, 2, 3} + envVisibleDevicesStr := make([]string, len(envVisibleDevices)) + for i, idx := range envVisibleDevices { + envVisibleDevicesStr[i] = strconv.Itoa(int(idx)) + } + envVisibleDevicesValue := strings.Join(envVisibleDevicesStr, ",") + + procFs := uprobes.CreateFakeProcFS(t, []uprobes.FakeProcFSEntry{ + {Pid: uint32(pidNoContainer)}, + {Pid: uint32(pidContainer)}, + {Pid: uint32(pidContainerAndEnv), Env: map[string]string{"CUDA_VISIBLE_DEVICES": envVisibleDevicesValue}}, + {Pid: uint32(pidNoContainerButEnv), Env: map[string]string{"CUDA_VISIBLE_DEVICES": envVisibleDevicesValue}}, + }) + + wmetaMock := testutil.GetWorkloadMetaMock(t) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), procFs, wmetaMock) + require.NotNil(t, sysCtx) + require.NoError(t, err) + + // Create a container with a single GPU and add it to the store + containerID := "abcdef" + container := &workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindContainer, + ID: containerID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: containerID, + }, + } + + containerDeviceIndexes := []int32{1, 2, 3, 4} + for _, idx := range containerDeviceIndexes { + gpuUUID := testutil.GPUUUIDs[idx] + resource := workloadmeta.ContainerAllocatedResource{ + Name: nvidiaResourceName, + ID: gpuUUID, + } + container.AllocatedResources = append(container.AllocatedResources, resource) + } + + wmetaMock.Set(container) + storeContainer, err := wmetaMock.GetContainer(containerID) + require.NoError(t, err, "container should be found in the store") + require.NotNil(t, storeContainer, "container should be found in the store") + + cases := []struct { + name string + pid int + containerID string + configuredDeviceIdx []int32 + expectedDeviceIdx []int32 + }{ + { + name: "NoContainer", + containerID: "", + pid: pidNoContainer, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{1, 2}, + }, + { + name: "NoContainerButEnv", + containerID: "", + pid: pidNoContainerButEnv, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{envVisibleDevices[1], envVisibleDevices[2]}, + }, + { + name: "WithContainer", + containerID: containerID, + pid: pidContainer, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{containerDeviceIndexes[1], containerDeviceIndexes[2]}, + }, + { + name: "WithContainerAndEnv", + pid: pidContainerAndEnv, + containerID: containerID, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{containerDeviceIndexes[envVisibleDevices[1]], containerDeviceIndexes[envVisibleDevices[2]]}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + for i, idx := range c.configuredDeviceIdx { + sysCtx.setDeviceSelection(c.pid, c.pid+i, idx) + } + + for i, idx := range c.expectedDeviceIdx { + activeDevice, err := sysCtx.getCurrentActiveGpuDevice(c.pid, c.pid+i, c.containerID) + require.NoError(t, err) + testutil.RequireDevicesEqual(t, sysCtx.gpuDevices[idx], activeDevice, "invalid device at index %d (real index is %d, selected index is %d)", i, idx, c.configuredDeviceIdx[i]) + } + }) + } +} diff --git a/pkg/gpu/probe.go b/pkg/gpu/probe.go index 3ab58ede54f35..fafc90df19c28 100644 --- a/pkg/gpu/probe.go +++ b/pkg/gpu/probe.go @@ -22,6 +22,7 @@ import ( "github.com/cilium/ebpf" "github.com/DataDog/datadog-agent/comp/core/telemetry" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" ddebpf "github.com/DataDog/datadog-agent/pkg/ebpf" ebpftelemetry "github.com/DataDog/datadog-agent/pkg/ebpf/telemetry" @@ -81,6 +82,10 @@ type ProbeDependencies struct { // ProcessMonitor is the process monitor interface ProcessMonitor uprobes.ProcessMonitor + + // WorkloadMeta used to retrieve data about workloads (containers, processes) running + // on the host + WorkloadMeta workloadmeta.Component } // Probe represents the GPU monitoring probe @@ -109,7 +114,7 @@ func NewProbe(cfg *config.Config, deps ProbeDependencies) (*Probe, error) { } attachCfg := getAttacherConfig(cfg) - sysCtx, err := getSystemContext(deps.NvmlLib, cfg.ProcRoot) + sysCtx, err := getSystemContext(deps.NvmlLib, cfg.ProcRoot, deps.WorkloadMeta) if err != nil { return nil, fmt.Errorf("error getting system context: %w", err) } diff --git a/pkg/gpu/probe_stub.go b/pkg/gpu/probe_stub.go index 734a11919abb9..2f4ebf544c984 100644 --- a/pkg/gpu/probe_stub.go +++ b/pkg/gpu/probe_stub.go @@ -11,6 +11,7 @@ import ( "github.com/NVIDIA/go-nvml/pkg/nvml" "github.com/DataDog/datadog-agent/comp/core/telemetry" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" "github.com/DataDog/datadog-agent/pkg/ebpf" "github.com/DataDog/datadog-agent/pkg/gpu/config" @@ -21,6 +22,7 @@ type ProbeDependencies struct { Telemetry telemetry.Component NvmlLib nvml.Interface ProcessMonitor any // uprobes.ProcessMonitor is only compiled with the linux_bpf build tag, so we need to use type any here + WorkloadMeta workloadmeta.Component } // Probe is not implemented on non-linux systems diff --git a/pkg/gpu/probe_test.go b/pkg/gpu/probe_test.go index f3eda56e2d2d9..ff2834b0327bc 100644 --- a/pkg/gpu/probe_test.go +++ b/pkg/gpu/probe_test.go @@ -47,6 +47,7 @@ func (s *probeTestSuite) getProbe() *Probe { deps := ProbeDependencies{ NvmlLib: testutil.GetBasicNvmlMock(), ProcessMonitor: consumerstestutil.NewTestProcessConsumer(t), + WorkloadMeta: testutil.GetWorkloadMetaMock(t), } probe, err := NewProbe(cfg, deps) require.NoError(t, err) diff --git a/pkg/gpu/stats_test.go b/pkg/gpu/stats_test.go index c0445e6c4e869..5aa4b2304cf3a 100644 --- a/pkg/gpu/stats_test.go +++ b/pkg/gpu/stats_test.go @@ -31,7 +31,7 @@ func getMetricsEntry(key model.StatsKey, stats *model.GPUStats) *model.Utilizati } func getStatsGeneratorForTest(t *testing.T) (*statsGenerator, map[streamKey]*StreamHandler, int64) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) require.NotNil(t, sysCtx) diff --git a/pkg/gpu/stream_test.go b/pkg/gpu/stream_test.go index f33e4f342e98b..0e4f0a5b734c8 100644 --- a/pkg/gpu/stream_test.go +++ b/pkg/gpu/stream_test.go @@ -21,10 +21,16 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/kernel" ) -func TestKernelLaunchesHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) +func getSystemContextForTest(t *testing.T) *systemContext { + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + require.NotNil(t, sysCtx) + + return sysCtx +} + +func TestKernelLaunchesHandled(t *testing.T) { + stream := newStreamHandler(0, "", getSystemContextForTest(t)) kernStartTime := uint64(1) launch := &gpuebpf.CudaKernelLaunch{ @@ -81,9 +87,7 @@ func TestKernelLaunchesHandled(t *testing.T) { } func TestMemoryAllocationsHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memFreeTime := uint64(2) @@ -152,9 +156,7 @@ func TestMemoryAllocationsHandled(t *testing.T) { } func TestMemoryAllocationsDetectLeaks(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memAddr := uint64(42) @@ -187,9 +189,7 @@ func TestMemoryAllocationsDetectLeaks(t *testing.T) { } func TestMemoryAllocationsNoCrashOnInvalidFree(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memFreeTime := uint64(2) @@ -231,9 +231,7 @@ func TestMemoryAllocationsNoCrashOnInvalidFree(t *testing.T) { } func TestMemoryAllocationsMultipleAllocsHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime1, memAllocTime2 := uint64(1), uint64(10) memFreeTime1, memFreeTime2 := uint64(15), uint64(20) @@ -324,7 +322,7 @@ func TestMemoryAllocationsMultipleAllocsHandled(t *testing.T) { func TestKernelLaunchesIncludeEnrichedKernelData(t *testing.T) { proc := kernel.ProcFSRoot() - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), proc) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), proc, testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) // Set up the caches in system context so no actual queries are done diff --git a/pkg/gpu/testutil/mocks.go b/pkg/gpu/testutil/mocks.go index a7dadd39215ad..009385125600a 100644 --- a/pkg/gpu/testutil/mocks.go +++ b/pkg/gpu/testutil/mocks.go @@ -9,8 +9,19 @@ package testutil import ( + "fmt" + "testing" + "github.com/NVIDIA/go-nvml/pkg/nvml" nvmlmock "github.com/NVIDIA/go-nvml/pkg/nvml/mock" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + + "github.com/DataDog/datadog-agent/comp/core" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + workloadmetafxmock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx-mock" + workloadmetamock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) // DefaultGpuCores is the default number of cores for a GPU device in the mock. @@ -18,13 +29,18 @@ const DefaultGpuCores = 10 // GPUUUIDs is a list of UUIDs for the devices returned by the mock var GPUUUIDs = []string{ - "GPU-12345678-1234-1234-1234-123456789012", - "GPU-99999999-1234-1234-1234-123456789013", - "GPU-00000000-1234-1234-1234-123456789014", + "GPU-00000000-1234-1234-1234-123456789012", + "GPU-11111111-1234-1234-1234-123456789013", + "GPU-22222222-1234-1234-1234-123456789014", + "GPU-33333333-1234-1234-1234-123456789015", + "GPU-44444444-1234-1234-1234-123456789016", + "GPU-55555555-1234-1234-1234-123456789017", + "GPU-66666666-1234-1234-1234-123456789018", } -// GPUCores is a list of number of cores for the devices returned by the mock, should be the same length as GPUUUIDs -var GPUCores = []int{DefaultGpuCores, 20, 30} +// GPUCores is a list of number of cores for the devices returned by the mock, +// should be the same length as GPUUUIDs. If not, GetBasicNvmlMock will panic. +var GPUCores = []int{DefaultGpuCores, 20, 30, 40, 50, 60, 70} // DefaultGpuUUID is the UUID for the default device returned by the mock var DefaultGpuUUID = GPUUUIDs[0] @@ -47,6 +63,11 @@ func GetDeviceMock(deviceIdx int) *nvmlmock.Device { // GetBasicNvmlMock returns a mock of the nvml.Interface with a single device with 10 cores, // useful for basic tests that need only the basic interaction with NVML to be working. func GetBasicNvmlMock() *nvmlmock.Interface { + if len(GPUUUIDs) != len(GPUCores) { + // Make it really easy to spot errors if we change any of the arrays. + panic("GPUUUIDs and GPUCores must have the same length, please fix it") + } + return &nvmlmock.Interface{ DeviceGetCountFunc: func() (int, nvml.Return) { return len(GPUUUIDs), nvml.SUCCESS @@ -59,3 +80,49 @@ func GetBasicNvmlMock() *nvmlmock.Interface { }, } } + +// GetWorkloadMetaMock returns a mock of the workloadmeta.Component. +func GetWorkloadMetaMock(t *testing.T) workloadmetamock.Mock { + return fxutil.Test[workloadmetamock.Mock](t, fx.Options( + core.MockBundle(), + workloadmetafxmock.MockModule(workloadmeta.NewParams()), + )) +} + +// RequireDevicesEqual checks that the two devices are equal by comparing their UUIDs, which gives a better +// output than using require.Equal on the devices themselves +func RequireDevicesEqual(t *testing.T, expected, actual nvml.Device, msgAndArgs ...interface{}) { + extraFmt := "" + if len(msgAndArgs) > 0 { + extraFmt = fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + ": " + } + + expectedUUID, ret := expected.GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%s%scannot retrieve UUID for expected device %v%s", extraFmt, expected) + + actualUUID, ret := actual.GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for actual device %v%s", extraFmt, actual) + + require.Equal(t, expectedUUID, actualUUID, "%sUUIDs do not match", extraFmt) +} + +// RequireDeviceListsEqual checks that the two device lists are equal by comparing their UUIDs, which gives a better +// output than using require.ElementsMatch on the lists themselves +func RequireDeviceListsEqual(t *testing.T, expected, actual []nvml.Device, msgAndArgs ...interface{}) { + extraFmt := "" + if len(msgAndArgs) > 0 { + extraFmt = fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + ": " + } + + require.Len(t, actual, len(expected), "%sdevice lists have different lengths", extraFmt) + + for i := range expected { + expectedUUID, ret := expected[i].GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for expected device index %d", extraFmt, i) + + actualUUID, ret := actual[i].GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for actual device index %d", extraFmt, i) + + require.Equal(t, expectedUUID, actualUUID, "%sUUIDs do not match for element %d", extraFmt, i) + } +} diff --git a/pkg/internaltelemetry/client.go b/pkg/internaltelemetry/client.go index 52b1b2010edf0..ee7754fad190b 100644 --- a/pkg/internaltelemetry/client.go +++ b/pkg/internaltelemetry/client.go @@ -21,8 +21,6 @@ import ( "go.uber.org/atomic" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - "github.com/DataDog/datadog-agent/pkg/trace/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" "github.com/shirou/gopsutil/v4/host" @@ -36,13 +34,19 @@ const ( // Client defines the interface for a telemetry client type Client interface { SendLog(level string, message string) - SendTraces(traces pb.Traces) + SendTraces(traces Traces) +} + +// Endpoint defines the endpoint object +type Endpoint struct { + APIKey string `json:"-"` + Host string } type client struct { m sync.Mutex client httpClient - endpoints []*config.Endpoint + endpoints []*Endpoint sendPayloadTimeout time.Duration // we can pre-calculate the host payload structure at init time @@ -94,7 +98,7 @@ type Application struct { // TracePayload defines the trace payload object type TracePayload struct { - Traces []pb.Trace `json:"traces"` + Traces []Trace `json:"traces"` } // LogPayload defines the log payload object @@ -115,7 +119,7 @@ type httpClient interface { } // NewClient creates a new telemetry client -func NewClient(httpClient httpClient, endpoints []*config.Endpoint, service string, debug bool) Client { +func NewClient(httpClient httpClient, endpoints []*Endpoint, service string, debug bool) Client { info, err := host.Info() if err != nil { log.Errorf("failed to retrieve host info: %v", err) @@ -158,7 +162,7 @@ func (c *client) SendLog(level, message string) { c.sendPayload(RequestTypeLogs, payload) } -func (c *client) SendTraces(traces pb.Traces) { +func (c *client) SendTraces(traces Traces) { c.m.Lock() defer c.m.Unlock() payload := TracePayload{ @@ -170,8 +174,8 @@ func (c *client) SendTraces(traces pb.Traces) { // sampleTraces is a simple uniform sampling function that samples traces based // on the sampling rate, given that there is no trace agent to sample the traces // We try to keep the tracer behaviour: the first rule that matches apply its rate to the whole trace -func (c *client) sampleTraces(traces pb.Traces) pb.Traces { - tracesWithSampling := pb.Traces{} +func (c *client) sampleTraces(traces Traces) Traces { + tracesWithSampling := Traces{} for _, trace := range traces { samplingRate := 1.0 for _, span := range trace { @@ -206,7 +210,7 @@ func (c *client) sendPayload(requestType RequestType, payload interface{}) { group := sync.WaitGroup{} for _, endpoint := range c.endpoints { group.Add(1) - go func(endpoint *config.Endpoint) { + go func(endpoint *Endpoint) { defer group.Done() url := fmt.Sprintf("%s%s", endpoint.Host, telemetryEndpoint) req, err := http.NewRequest("POST", url, bytes.NewReader(serializedPayload)) diff --git a/pkg/internaltelemetry/traces.go b/pkg/internaltelemetry/traces.go new file mode 100644 index 0000000000000..54c8545a604b7 --- /dev/null +++ b/pkg/internaltelemetry/traces.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +// Package internaltelemetry full description in README.md +package internaltelemetry + +// Traces is a collection of traces +type Traces []Trace + +// Trace is a collection of spans with the same trace ID +type Trace []*Span + +// Span used for installation telemetry +type Span struct { + // Service is the name of the service that handled this span. + Service string `json:"service"` + // Name is the name of the operation this span represents. + Name string `json:"name"` + // Resource is the name of the resource this span represents. + Resource string `json:"resource"` + // TraceID is the ID of the trace to which this span belongs. + TraceID uint64 `json:"trace_id"` + // SpanID is the ID of this span. + SpanID uint64 `json:"span_id"` + // ParentID is the ID of the parent span. + ParentID uint64 `json:"parent_id"` + // Start is the start time of this span in nanoseconds since the Unix epoch. + Start int64 `json:"start"` + // Duration is the duration of this span in nanoseconds. + Duration int64 `json:"duration"` + // Error is the error status of this span. + Error int32 `json:"error"` + // Meta is a mapping from tag name to tag value for string-valued tags. + Meta map[string]string `json:"meta,omitempty"` + // Metrics is a mapping from metric name to metric value for numeric metrics. + Metrics map[string]float64 `json:"metrics,omitempty"` + // Type is the type of the span. + Type string `json:"type"` +} diff --git a/pkg/kubestatemetrics/builder/builder.go b/pkg/kubestatemetrics/builder/builder.go index d0c5ebd1ea998..ea26602567f56 100644 --- a/pkg/kubestatemetrics/builder/builder.go +++ b/pkg/kubestatemetrics/builder/builder.go @@ -9,6 +9,7 @@ package builder import ( "context" + "fmt" "reflect" "time" @@ -18,7 +19,11 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + apiwatch "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" ksmbuild "k8s.io/kube-state-metrics/v2/pkg/builder" ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" @@ -196,6 +201,14 @@ func GenerateStores[T any]( isPod = true } else if u, ok := expectedType.(*unstructured.Unstructured); ok { isPod = u.GetAPIVersion() == "v1" && u.GetKind() == "Pod" + } else if _, ok := expectedType.(*corev1.ConfigMap); ok { + configMapStore, err := generateConfigMapStores(b, metricFamilies, useAPIServerCache) + if err != nil { + log.Debugf("Defaulting to kube-state-metrics for configmap collection: %v", err) + } else { + log.Debug("Using meta.k8s.io API for configmap collection") + return configMapStore + } } if b.namespaces.IsAllNamespaces() { @@ -340,3 +353,101 @@ func handlePodCollection[T any](b *Builder, store cache.Store, client T, listWat listWatcher := listWatchFunc(client, namespace, fieldSelector) b.startReflector(&corev1.Pod{}, store, listWatcher, useAPIServerCache) } + +func generateConfigMapStores( + b *Builder, + metricFamilies []generator.FamilyGenerator, + useAPIServerCache bool, +) ([]cache.Store, error) { + restConfig, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to create in-cluster config for metadata client: %w", err) + } + + metadataClient, err := metadata.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create metadata client: %w", err) + } + + gvr := schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + } + + filteredMetricFamilies := generator.FilterFamilyGenerators(b.allowDenyList, metricFamilies) + composedMetricGenFuncs := generator.ComposeMetricGenFuncs(filteredMetricFamilies) + + stores := make([]cache.Store, 0) + + if b.namespaces.IsAllNamespaces() { + log.Infof("Using NamespaceAll for ConfigMap collection.") + store := store.NewMetricsStore(composedMetricGenFuncs, "configmap") + listWatcher := createConfigMapListWatch(metadataClient, gvr, v1.NamespaceAll) + b.startReflector(&corev1.ConfigMap{}, store, listWatcher, useAPIServerCache) + return []cache.Store{store}, nil + } + + for _, ns := range b.namespaces { + store := store.NewMetricsStore(composedMetricGenFuncs, "configmap") + listWatcher := createConfigMapListWatch(metadataClient, gvr, ns) + b.startReflector(&corev1.ConfigMap{}, store, listWatcher, useAPIServerCache) + stores = append(stores, store) + } + + return stores, nil +} + +func createConfigMapListWatch(metadataClient metadata.Interface, gvr schema.GroupVersionResource, namespace string) *cache.ListWatch { + return &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + result, err := metadataClient.Resource(gvr).Namespace(namespace).List(context.TODO(), options) + if err != nil { + return nil, err + } + + configMapList := &corev1.ConfigMapList{} + for _, item := range result.Items { + configMapList.Items = append(configMapList.Items, corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + UID: item.GetUID(), + ResourceVersion: item.GetResourceVersion(), + }, + }) + } + + return configMapList, nil + }, + WatchFunc: func(options v1.ListOptions) (apiwatch.Interface, error) { + watcher, err := metadataClient.Resource(gvr).Namespace(namespace).Watch(context.TODO(), options) + if err != nil { + return nil, err + } + + return apiwatch.Filter(watcher, func(event apiwatch.Event) (apiwatch.Event, bool) { + if event.Object == nil { + return event, false + } + + partialObject, ok := event.Object.(*v1.PartialObjectMetadata) + if !ok { + return event, false + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: partialObject.GetName(), + Namespace: partialObject.GetNamespace(), + UID: partialObject.GetUID(), + ResourceVersion: partialObject.GetResourceVersion(), + }, + } + + event.Object = configMap + return event, true + }), nil + }, + } +} diff --git a/pkg/logs/auditor/go.mod b/pkg/logs/auditor/go.mod index 3fc9bf65dfa46..f9543ab0dda08 100644 --- a/pkg/logs/auditor/go.mod +++ b/pkg/logs/auditor/go.mod @@ -80,7 +80,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/logs/auditor/go.sum b/pkg/logs/auditor/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/logs/auditor/go.sum +++ b/pkg/logs/auditor/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/logs/client/go.mod b/pkg/logs/client/go.mod index 093922b236bf3..07b8c9ec2e3be 100644 --- a/pkg/logs/client/go.mod +++ b/pkg/logs/client/go.mod @@ -63,7 +63,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 ) require ( @@ -100,7 +100,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect diff --git a/pkg/logs/client/go.sum b/pkg/logs/client/go.sum index 1140bccbba9c9..b14b43571558e 100644 --- a/pkg/logs/client/go.sum +++ b/pkg/logs/client/go.sum @@ -88,8 +88,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -263,8 +263,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/logs/client/http/destination.go b/pkg/logs/client/http/destination.go index 2a554e5476504..93bf35ec67167 100644 --- a/pkg/logs/client/http/destination.go +++ b/pkg/logs/client/http/destination.go @@ -396,6 +396,13 @@ func httpClientFactory(timeout time.Duration, cfg pkgconfigmodel.Reader) func() var transport *http.Transport transportConfig := cfg.Get("logs_config.http_protocol") + + // If any proxy is set, use http1 + // This will be removed in a future version + if cfg.GetProxies() != nil { + transportConfig = "http1" + } + // Configure transport based on user setting switch transportConfig { case "http1": diff --git a/pkg/logs/client/http/destination_test.go b/pkg/logs/client/http/destination_test.go index 92bf984a79e5a..3f65bba9b9e2c 100644 --- a/pkg/logs/client/http/destination_test.go +++ b/pkg/logs/client/http/destination_test.go @@ -494,3 +494,34 @@ func TestTransportProtocol_HTTP1FallBack(t *testing.T) { // Assert that the server automatically falls back to HTTP/1.1 assert.Equal(t, "HTTP/1.1", resp.Proto) } + +func TestTransportProtocol_HTTP1WhenUsingProxy(t *testing.T) { + c := configmock.New(t) + + // Force client to use ALNP + c.SetWithoutSource("logs_config.http_protocol", "auto") + c.SetWithoutSource("skip_ssl_validation", true) + + // The test server uses TLS, so if we set the http proxy (not https), it still makes + // a request to the test server, but disable HTTP/2 since a proxy is configured. + c.SetWithoutSource("proxy.http", "http://foo.bar") + + server := NewTestHTTPSServer(false) + defer server.Close() + + timeout := 5 * time.Second + client := httpClientFactory(timeout, c)() + + req, err := http.NewRequest("POST", server.URL, nil) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to send request: %v", err) + } + defer resp.Body.Close() + + // Assert that the server chose HTTP/1.1 because a proxy was configured + assert.Equal(t, "HTTP/1.1", resp.Proto) +} diff --git a/pkg/logs/diagnostic/go.mod b/pkg/logs/diagnostic/go.mod index 65fb4af75f769..690eb4c56105e 100644 --- a/pkg/logs/diagnostic/go.mod +++ b/pkg/logs/diagnostic/go.mod @@ -84,7 +84,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect diff --git a/pkg/logs/diagnostic/go.sum b/pkg/logs/diagnostic/go.sum index 9694f5d5e2355..dd16364891695 100644 --- a/pkg/logs/diagnostic/go.sum +++ b/pkg/logs/diagnostic/go.sum @@ -87,8 +87,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/logs/message/go.mod b/pkg/logs/message/go.mod index 716c4a57de92f..93c9793bafaf5 100644 --- a/pkg/logs/message/go.mod +++ b/pkg/logs/message/go.mod @@ -76,7 +76,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/logs/message/go.sum b/pkg/logs/message/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/logs/message/go.sum +++ b/pkg/logs/message/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/logs/pipeline/go.mod b/pkg/logs/pipeline/go.mod index b2cca0985481e..dbb50c6aec2e7 100644 --- a/pkg/logs/pipeline/go.mod +++ b/pkg/logs/pipeline/go.mod @@ -120,7 +120,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -150,7 +150,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/pkg/logs/pipeline/go.sum b/pkg/logs/pipeline/go.sum index 9061e9b89a5c6..19075c11eaba4 100644 --- a/pkg/logs/pipeline/go.sum +++ b/pkg/logs/pipeline/go.sum @@ -99,8 +99,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/logs/processor/go.mod b/pkg/logs/processor/go.mod index cc9458ea710c0..b98eacd627e97 100644 --- a/pkg/logs/processor/go.mod +++ b/pkg/logs/processor/go.mod @@ -99,7 +99,7 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect diff --git a/pkg/logs/processor/go.sum b/pkg/logs/processor/go.sum index f97a65425513e..1fb7bbe76cabd 100644 --- a/pkg/logs/processor/go.sum +++ b/pkg/logs/processor/go.sum @@ -94,8 +94,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/logs/sds/go.mod b/pkg/logs/sds/go.mod index 33c40479ed13f..f5818700af57b 100644 --- a/pkg/logs/sds/go.mod +++ b/pkg/logs/sds/go.mod @@ -93,7 +93,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect diff --git a/pkg/logs/sds/go.sum b/pkg/logs/sds/go.sum index a5a699a22e560..c9341f0f48498 100644 --- a/pkg/logs/sds/go.sum +++ b/pkg/logs/sds/go.sum @@ -88,8 +88,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/logs/sender/go.mod b/pkg/logs/sender/go.mod index 5a77fb5bb761d..28b1f5738002a 100644 --- a/pkg/logs/sender/go.mod +++ b/pkg/logs/sender/go.mod @@ -100,7 +100,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -131,7 +131,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/pkg/logs/sender/go.sum b/pkg/logs/sender/go.sum index 1140bccbba9c9..b14b43571558e 100644 --- a/pkg/logs/sender/go.sum +++ b/pkg/logs/sender/go.sum @@ -88,8 +88,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -263,8 +263,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/logs/sources/go.mod b/pkg/logs/sources/go.mod index a5ff5d7147d72..47c016d543c91 100644 --- a/pkg/logs/sources/go.mod +++ b/pkg/logs/sources/go.mod @@ -74,7 +74,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/logs/sources/go.sum b/pkg/logs/sources/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/logs/sources/go.sum +++ b/pkg/logs/sources/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/logs/util/testutils/go.mod b/pkg/logs/util/testutils/go.mod index ebf193c2c90a2..1c02f6f2f8775 100644 --- a/pkg/logs/util/testutils/go.mod +++ b/pkg/logs/util/testutils/go.mod @@ -74,7 +74,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -95,4 +95,5 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/logs/util/testutils/go.sum b/pkg/logs/util/testutils/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/logs/util/testutils/go.sum +++ b/pkg/logs/util/testutils/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/metrics/go.mod b/pkg/metrics/go.mod index a7369cfd728f6..66e73954c2885 100644 --- a/pkg/metrics/go.mod +++ b/pkg/metrics/go.mod @@ -85,7 +85,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect diff --git a/pkg/metrics/go.sum b/pkg/metrics/go.sum index cc70504832885..31112092709e6 100644 --- a/pkg/metrics/go.sum +++ b/pkg/metrics/go.sum @@ -96,8 +96,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/metrics/metricsource.go b/pkg/metrics/metricsource.go index f3afa888d60b4..5fcc0ff956952 100644 --- a/pkg/metrics/metricsource.go +++ b/pkg/metrics/metricsource.go @@ -308,6 +308,7 @@ const ( MetricSourceAnyscale MetricSourceMilvus MetricSourceNvidiaNim + MetricSourceQuarkus ) // String returns a string representation of MetricSource @@ -879,6 +880,8 @@ func (ms MetricSource) String() string { return "aws_neuron" case MetricSourceMilvus: return "milvus" + case MetricSourceQuarkus: + return "quarkus" default: return "" } @@ -1421,6 +1424,8 @@ func CheckNameToMetricSource(name string) MetricSource { return MetricSourceTibcoEMS case "milvus": return MetricSourceMilvus + case "quarkus": + return MetricSourceQuarkus default: return MetricSourceUnknown } diff --git a/pkg/network/ebpf/c/co-re/tracer-fentry.c b/pkg/network/ebpf/c/co-re/tracer-fentry.c index 09259e948047d..b89ab7e351d35 100644 --- a/pkg/network/ebpf/c/co-re/tracer-fentry.c +++ b/pkg/network/ebpf/c/co-re/tracer-fentry.c @@ -8,6 +8,7 @@ #include "ipv6.h" #include "sock.h" #include "skb.h" +#include "pid_tgid.h" #include "tracer/tracer.h" #include "tracer/events.h" @@ -52,7 +53,7 @@ static __always_inline bool event_in_task(char *prog_name) { } static __always_inline int read_conn_tuple_partial_from_flowi4(conn_tuple_t *t, struct flowi4 *fl4, u64 pid_tgid, metadata_mask_t type) { - t->pid = pid_tgid >> 32; + t->pid = GET_USER_MODE_PID(pid_tgid); t->metadata = type; if (t->saddr_l == 0) { @@ -85,7 +86,7 @@ static __always_inline int read_conn_tuple_partial_from_flowi4(conn_tuple_t *t, } static __always_inline int read_conn_tuple_partial_from_flowi6(conn_tuple_t *t, struct flowi6 *fl6, u64 pid_tgid, metadata_mask_t type) { - t->pid = pid_tgid >> 32; + t->pid = GET_USER_MODE_PID(pid_tgid); t->metadata = type; struct in6_addr addr = BPF_CORE_READ(fl6, saddr); @@ -233,7 +234,7 @@ int BPF_PROG(tcp_close, struct sock *sk, long timeout) { u64 pid_tgid = bpf_get_current_pid_tgid(); // Get network namespace id - log_debug("fentry/tcp_close: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + log_debug("fentry/tcp_close: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); if (!read_conn_tuple(&t, sk, pid_tgid, CONN_TYPE_TCP)) { return 0; } @@ -410,14 +411,14 @@ SEC("fentry/tcp_retransmit_skb") int BPF_PROG(tcp_retransmit_skb, struct sock *sk, struct sk_buff *skb, int segs, int err) { RETURN_IF_NOT_IN_SYSPROBE_TASK("fentry/tcp_retransmit_skb"); log_debug("fexntry/tcp_retransmit"); - u64 tid = bpf_get_current_pid_tgid(); + u64 pid_tgid = bpf_get_current_pid_tgid(); tcp_retransmit_skb_args_t args = {}; args.retrans_out_pre = BPF_CORE_READ(tcp_sk(sk), retrans_out); if (args.retrans_out_pre < 0) { return 0; } - bpf_map_update_with_telemetry(pending_tcp_retransmit_skb, &tid, &args, BPF_ANY); + bpf_map_update_with_telemetry(pending_tcp_retransmit_skb, &pid_tgid, &args, BPF_ANY); return 0; } @@ -426,18 +427,18 @@ SEC("fexit/tcp_retransmit_skb") int BPF_PROG(tcp_retransmit_skb_exit, struct sock *sk, struct sk_buff *skb, int segs, int err) { RETURN_IF_NOT_IN_SYSPROBE_TASK("fexit/tcp_retransmit_skb"); log_debug("fexit/tcp_retransmit"); - u64 tid = bpf_get_current_pid_tgid(); + u64 pid_tgid = bpf_get_current_pid_tgid(); if (err < 0) { - bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid); + bpf_map_delete_elem(&pending_tcp_retransmit_skb, &pid_tgid); return 0; } - tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &tid); + tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &pid_tgid); if (args == NULL) { return 0; } u32 retrans_out_pre = args->retrans_out_pre; u32 retrans_out = BPF_CORE_READ(tcp_sk(sk), retrans_out); - bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid); + bpf_map_delete_elem(&pending_tcp_retransmit_skb, &pid_tgid); if (retrans_out < 0) { return 0; @@ -450,7 +451,7 @@ SEC("fentry/tcp_connect") int BPF_PROG(tcp_connect, struct sock *sk) { RETURN_IF_NOT_IN_SYSPROBE_TASK("fentry/tcp_connect"); u64 pid_tgid = bpf_get_current_pid_tgid(); - log_debug("fentry/tcp_connect: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + log_debug("fentry/tcp_connect: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); conn_tuple_t t = {}; if (!read_conn_tuple(&t, sk, 0, CONN_TYPE_TCP)) { @@ -479,8 +480,8 @@ int BPF_PROG(tcp_finish_connect, struct sock *sk, struct sk_buff *skb, int rc) { return 0; } u64 pid_tgid = pid_tgid_p->pid_tgid; - t.pid = pid_tgid >> 32; - log_debug("fentry/tcp_finish_connect: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + t.pid = GET_USER_MODE_PID(pid_tgid); + log_debug("fentry/tcp_finish_connect: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); handle_tcp_stats(&t, sk, TCP_ESTABLISHED); handle_message(&t, 0, 0, CONN_DIRECTION_OUTGOING, 0, 0, PACKET_COUNT_NONE, sk); @@ -498,7 +499,7 @@ int BPF_PROG(inet_csk_accept_exit, struct sock *_sk, int flags, int *err, bool k } u64 pid_tgid = bpf_get_current_pid_tgid(); - log_debug("fexit/inet_csk_accept: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + log_debug("fexit/inet_csk_accept: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); conn_tuple_t t = {}; if (!read_conn_tuple(&t, sk, pid_tgid, CONN_TYPE_TCP)) { diff --git a/pkg/network/ebpf/c/pid_tgid.h b/pkg/network/ebpf/c/pid_tgid.h new file mode 100644 index 0000000000000..a3439bca233a7 --- /dev/null +++ b/pkg/network/ebpf/c/pid_tgid.h @@ -0,0 +1,52 @@ +#ifndef __PID_TGID_H +#define __PID_TGID_H + +/* + * The following documentation is based on https://stackoverflow.com/a/9306150 + * Note on Process and Thread Identifiers: + * + * What users refer to as a "PID" is not quite the same as what the kernel sees. + * + * In the kernel: + * - Each thread has its own ID, called a PID (though it might be better termed a TID, or Thread ID). + * - Threads within the same process share a TGID (Thread Group ID), which is the PID of the first thread + * created when the process was initialized. + * + * When a process is created: + * - It starts as a single thread where the PID and TGID are the same. + * + * When a new thread is created: + * - It receives its own unique PID for independent scheduling by the kernel. + * - It inherits the TGID from the original (parent) thread, tying it to the same process. + * + * This separation allows the kernel to schedule threads independently while maintaining the process view + * (TGID) when reporting information to users. + * + * Example Hierarchy of Threads: + * + * USER VIEW + * vvvvvvvv + * + * | + * <-- PID 43 -->|<----------------- PID 42 -----------------> + * | | + * | +---------+ | + * | | process | | + * | _| pid=42 |_ | + * __(fork) _/ | tgid=42 | \_ (new thread) _ + * / | +---------+ | \ + * +---------+ | | +---------+ + * | process | | | | process | + * | pid=43 | | | | pid=44 | + * | tgid=43 | | | | tgid=42 | + * +---------+ | | +---------+ + * | | + * <-- PID 43 -->|<--------- PID 42 -------->|<--- PID 44 ---> + * | | + * ^^^^^^^^ + * KERNEL VIEW + */ +#define GET_USER_MODE_PID(x) ((x) >> 32) +#define GET_KERNEL_THREAD_ID(x) ((x) & 0xFFFFFFFF) + +#endif // __PID_TGID_H diff --git a/pkg/network/ebpf/c/prebuilt/conntrack.c b/pkg/network/ebpf/c/prebuilt/conntrack.c index 8e77c1fd98ff8..8417806ce3077 100644 --- a/pkg/network/ebpf/c/prebuilt/conntrack.c +++ b/pkg/network/ebpf/c/prebuilt/conntrack.c @@ -12,6 +12,7 @@ #include "conntrack/maps.h" #include "ip.h" #include "ipv6.h" +#include "pid_tgid.h" SEC("kprobe/__nf_conntrack_hash_insert") int BPF_BYPASSABLE_KPROBE(kprobe___nf_conntrack_hash_insert, struct nf_conn *ct) { @@ -32,7 +33,7 @@ int BPF_BYPASSABLE_KPROBE(kprobe___nf_conntrack_hash_insert, struct nf_conn *ct) SEC("kprobe/ctnetlink_fill_info") int BPF_BYPASSABLE_KPROBE(kprobe_ctnetlink_fill_info) { - u32 pid = bpf_get_current_pid_tgid() >> 32; + u32 pid = GET_USER_MODE_PID(bpf_get_current_pid_tgid()); if (pid != systemprobe_pid()) { log_debug("skipping kprobe/ctnetlink_fill_info invocation from non-system-probe process"); return 0; diff --git a/pkg/network/ebpf/c/prebuilt/offset-guess.c b/pkg/network/ebpf/c/prebuilt/offset-guess.c index cc32fee223935..556a770082684 100644 --- a/pkg/network/ebpf/c/prebuilt/offset-guess.c +++ b/pkg/network/ebpf/c/prebuilt/offset-guess.c @@ -298,11 +298,11 @@ int kprobe__sock_common_getsockopt(struct pt_regs* ctx) { SEC("kprobe/tcp_v6_connect") int kprobe__tcp_v6_connect(struct pt_regs* ctx) { struct sock* sk; - u64 pid = bpf_get_current_pid_tgid(); + u64 pid_tgid = bpf_get_current_pid_tgid(); sk = (struct sock*)PT_REGS_PARM1(ctx); - bpf_map_update_elem(&connectsock_ipv6, &pid, &sk, BPF_ANY); + bpf_map_update_elem(&connectsock_ipv6, &pid_tgid, &sk, BPF_ANY); return 0; } @@ -310,17 +310,17 @@ int kprobe__tcp_v6_connect(struct pt_regs* ctx) { // Used for offset guessing (see: pkg/ebpf/offsetguess.go) SEC("kretprobe/tcp_v6_connect") int kretprobe__tcp_v6_connect(struct pt_regs* __attribute__((unused)) ctx) { - u64 pid = bpf_get_current_pid_tgid(); + u64 pid_tgid = bpf_get_current_pid_tgid(); u64 zero = 0; struct sock** skpp; tracer_status_t* status; - skpp = bpf_map_lookup_elem(&connectsock_ipv6, &pid); + skpp = bpf_map_lookup_elem(&connectsock_ipv6, &pid_tgid); if (skpp == 0) { return 0; // missed entry } struct sock* skp = *skpp; - bpf_map_delete_elem(&connectsock_ipv6, &pid); + bpf_map_delete_elem(&connectsock_ipv6, &pid_tgid); status = bpf_map_lookup_elem(&tracer_status, &zero); if (status == NULL || is_sk_buff_event(status->what)) { diff --git a/pkg/network/ebpf/c/prebuilt/usm_events_test.c b/pkg/network/ebpf/c/prebuilt/usm_events_test.c index 820afb50ef0f0..f2bfb640aa54a 100644 --- a/pkg/network/ebpf/c/prebuilt/usm_events_test.c +++ b/pkg/network/ebpf/c/prebuilt/usm_events_test.c @@ -9,6 +9,7 @@ #include "defs.h" #include "map-defs.h" #include "protocols/events.h" +#include "pid_tgid.h" // -------------------------------------------------------- // this is a test program for pkg/networks/protocols/events @@ -37,7 +38,7 @@ struct syscalls_enter_write_args { SEC("tracepoint/syscalls/sys_enter_write") int tracepoint__syscalls__sys_enter_write(struct syscalls_enter_write_args *ctx) { __u32 zero = 0; - __u32 pid = bpf_get_current_pid_tgid() >> 32; + __u32 pid = GET_USER_MODE_PID(bpf_get_current_pid_tgid()); test_ctx_t *test_ctx = bpf_map_lookup_elem(&test, &zero); if (!test_ctx || test_ctx->expected_fd != ctx->fd || test_ctx->expected_pid != pid) return 0; diff --git a/pkg/network/ebpf/c/protocols/sockfd-probes.h b/pkg/network/ebpf/c/protocols/sockfd-probes.h index 9c31f236ab087..06ecbeee6fe57 100644 --- a/pkg/network/ebpf/c/protocols/sockfd-probes.h +++ b/pkg/network/ebpf/c/protocols/sockfd-probes.h @@ -13,6 +13,7 @@ #include "sock.h" #include "sockfd.h" +#include "pid_tgid.h" SEC("kprobe/tcp_close") int BPF_KPROBE(kprobe__tcp_close, struct sock *sk) { @@ -53,7 +54,7 @@ int BPF_KPROBE(kprobe__sockfd_lookup_light, int sockfd) { // but can reduce the accuracy of programs relying on socket FDs for // processes with a lot of FD churn pid_fd_t key = { - .pid = pid_tgid >> 32, + .pid = GET_USER_MODE_PID(pid_tgid), .fd = sockfd, }; conn_tuple_t *t = bpf_map_lookup_elem(&tuple_by_pid_fd, &key); @@ -121,7 +122,7 @@ int BPF_KRETPROBE(kretprobe__sockfd_lookup_light, struct socket *socket) { } pid_fd_t pid_fd = { - .pid = pid_tgid >> 32, + .pid = GET_USER_MODE_PID(pid_tgid), .fd = (*sockfd), }; diff --git a/pkg/network/ebpf/c/protocols/tls/go-tls-conn.h b/pkg/network/ebpf/c/protocols/tls/go-tls-conn.h index ba812de0385bd..d297ad11f57cf 100644 --- a/pkg/network/ebpf/c/protocols/tls/go-tls-conn.h +++ b/pkg/network/ebpf/c/protocols/tls/go-tls-conn.h @@ -4,6 +4,7 @@ #include "bpf_helpers.h" #include "ip.h" #include "port_range.h" +#include "pid_tgid.h" #include "protocols/http/maps.h" #include "protocols/tls/go-tls-types.h" @@ -49,7 +50,7 @@ static __always_inline conn_tuple_t* conn_tup_from_tls_conn(tls_offsets_data_t* // The code path below should be executed only once during the lifecycle of a TLS connection pid_fd_t pid_fd = { - .pid = pid_tgid >> 32, + .pid = GET_USER_MODE_PID(pid_tgid), // fd is populated by the code downstream .fd = 0, }; diff --git a/pkg/network/ebpf/c/protocols/tls/https.h b/pkg/network/ebpf/c/protocols/tls/https.h index f1d0562033a54..ac3c509daaeb3 100644 --- a/pkg/network/ebpf/c/protocols/tls/https.h +++ b/pkg/network/ebpf/c/protocols/tls/https.h @@ -17,6 +17,7 @@ #include "bpf_builtins.h" #include "port_range.h" #include "sock.h" +#include "pid_tgid.h" #include "protocols/amqp/helpers.h" #include "protocols/redis/helpers.h" @@ -250,7 +251,7 @@ static __always_inline conn_tuple_t* tup_from_ssl_ctx(void *ssl_ctx, u64 pid_tgi // the code path below should be executed only once during the lifecycle of a SSL session pid_fd_t pid_fd = { - .pid = pid_tgid >> 32, + .pid = GET_USER_MODE_PID(pid_tgid), .fd = ssl_sock->fd, }; diff --git a/pkg/network/ebpf/c/protocols/tls/native-tls-maps.h b/pkg/network/ebpf/c/protocols/tls/native-tls-maps.h index 282c593308b16..5e139833d4302 100644 --- a/pkg/network/ebpf/c/protocols/tls/native-tls-maps.h +++ b/pkg/network/ebpf/c/protocols/tls/native-tls-maps.h @@ -17,6 +17,6 @@ BPF_HASH_MAP(bio_new_socket_args, __u64, __u32, 1024) BPF_HASH_MAP(fd_by_ssl_bio, __u32, void *, 1024) -BPF_HASH_MAP(ssl_ctx_by_pid_tgid, __u64, void *, 1024) +BPF_HASH_MAP(ssl_ctx_by_pid_tgid, __u64, void *, 1) #endif diff --git a/pkg/network/ebpf/c/runtime/conntrack.c b/pkg/network/ebpf/c/runtime/conntrack.c index a054504439230..e7418edcd986e 100644 --- a/pkg/network/ebpf/c/runtime/conntrack.c +++ b/pkg/network/ebpf/c/runtime/conntrack.c @@ -21,6 +21,7 @@ #include "conntrack/maps.h" #include "netns.h" #include "ip.h" +#include "pid_tgid.h" #if defined(FEATURE_TCPV6_ENABLED) || defined(FEATURE_UDPV6_ENABLED) #include "ipv6.h" @@ -50,7 +51,7 @@ int BPF_BYPASSABLE_KPROBE(kprobe___nf_conntrack_hash_insert, struct nf_conn *ct) SEC("kprobe/ctnetlink_fill_info") int BPF_BYPASSABLE_KPROBE(kprobe_ctnetlink_fill_info) { - u32 pid = bpf_get_current_pid_tgid() >> 32; + u32 pid = GET_USER_MODE_PID(bpf_get_current_pid_tgid()); if (pid != systemprobe_pid()) { log_debug("skipping kprobe/ctnetlink_fill_info invocation from non-system-probe process"); return 0; diff --git a/pkg/network/ebpf/c/runtime/usm.c b/pkg/network/ebpf/c/runtime/usm.c index 3cb3f1245a3ab..bf3737b311a92 100644 --- a/pkg/network/ebpf/c/runtime/usm.c +++ b/pkg/network/ebpf/c/runtime/usm.c @@ -13,6 +13,7 @@ #include "ipv6.h" #include "sock.h" #include "port_range.h" +#include "pid_tgid.h" #include "protocols/classification/dispatcher-helpers.h" #include "protocols/http/buffer.h" @@ -84,7 +85,7 @@ int tracepoint__net__netif_receive_skb(void *ctx) { SEC("uprobe/crypto/tls.(*Conn).Write") int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Write) { u64 pid_tgid = bpf_get_current_pid_tgid(); - u64 pid = pid_tgid >> 32; + u64 pid = GET_USER_MODE_PID(pid_tgid); tls_offsets_data_t* od = get_offsets_data(); if (od == NULL) { log_debug("[go-tls-write] no offsets data in map for pid %llu", pid); @@ -125,7 +126,7 @@ int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Write) { SEC("uprobe/crypto/tls.(*Conn).Write/return") int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Write__return) { u64 pid_tgid = bpf_get_current_pid_tgid(); - u64 pid = pid_tgid >> 32; + u64 pid = GET_USER_MODE_PID(pid_tgid); tls_offsets_data_t* od = get_offsets_data(); if (od == NULL) { log_debug("[go-tls-write-return] no offsets data in map for pid %llu", pid); @@ -200,10 +201,10 @@ int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Write__return) { SEC("uprobe/crypto/tls.(*Conn).Read") int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Read) { u64 pid_tgid = bpf_get_current_pid_tgid(); - u64 pid = pid_tgid >> 32; + u64 pid = GET_USER_MODE_PID(pid_tgid); tls_offsets_data_t* od = get_offsets_data(); if (od == NULL) { - log_debug("[go-tls-read] no offsets data in map for pid %llu", pid_tgid >> 32); + log_debug("[go-tls-read] no offsets data in map for pid %llu", pid); return 0; } @@ -211,7 +212,7 @@ int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Read) { go_tls_function_args_key_t call_key = {0}; call_key.pid = pid; if (read_goroutine_id(ctx, &od->goroutine_id, &call_key.goroutine_id)) { - log_debug("[go-tls-read] failed reading go routine id for pid %llu", pid_tgid >> 32); + log_debug("[go-tls-read] failed reading go routine id for pid %llu", pid); return 0; } @@ -219,11 +220,11 @@ int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Read) { // (since the parameters might not be live by the time the return probe is hit). go_tls_read_args_data_t call_data = {0}; if (read_location(ctx, &od->read_conn_pointer, sizeof(call_data.conn_pointer), &call_data.conn_pointer)) { - log_debug("[go-tls-read] failed reading conn pointer for pid %llu", pid_tgid >> 32); + log_debug("[go-tls-read] failed reading conn pointer for pid %llu", pid); return 0; } if (read_location(ctx, &od->read_buffer.ptr, sizeof(call_data.b_data), &call_data.b_data)) { - log_debug("[go-tls-read] failed reading buffer pointer for pid %llu", pid_tgid >> 32); + log_debug("[go-tls-read] failed reading buffer pointer for pid %llu", pid); return 0; } @@ -235,7 +236,7 @@ int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Read) { SEC("uprobe/crypto/tls.(*Conn).Read/return") int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Read__return) { u64 pid_tgid = bpf_get_current_pid_tgid(); - u64 pid = pid_tgid >> 32; + u64 pid = GET_USER_MODE_PID(pid_tgid); tls_offsets_data_t* od = get_offsets_data(); if (od == NULL) { log_debug("[go-tls-read-return] no offsets data in map for pid %llu", pid); @@ -305,13 +306,13 @@ int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Close) { u64 pid_tgid = bpf_get_current_pid_tgid(); tls_offsets_data_t* od = get_offsets_data(); if (od == NULL) { - log_debug("[go-tls-close] no offsets data in map for pid %llu", pid_tgid >> 32); + log_debug("[go-tls-close] no offsets data in map for pid %llu", GET_USER_MODE_PID(pid_tgid)); return 0; } // Read the PID and goroutine ID to make the partial call key go_tls_function_args_key_t call_key = {0}; - call_key.pid = pid_tgid >> 32; + call_key.pid = GET_USER_MODE_PID(pid_tgid); if (read_goroutine_id(ctx, &od->goroutine_id, &call_key.goroutine_id) == 0) { bpf_map_delete_elem(&go_tls_read_args, &call_key); bpf_map_delete_elem(&go_tls_write_args, &call_key); @@ -319,13 +320,13 @@ int BPF_BYPASSABLE_UPROBE(uprobe__crypto_tls_Conn_Close) { void* conn_pointer = NULL; if (read_location(ctx, &od->close_conn_pointer, sizeof(conn_pointer), &conn_pointer)) { - log_debug("[go-tls-close] failed reading close conn pointer for pid %llu", pid_tgid >> 32); + log_debug("[go-tls-close] failed reading close conn pointer for pid %llu", GET_USER_MODE_PID(pid_tgid)); return 0; } conn_tuple_t* t = conn_tup_from_tls_conn(od, conn_pointer, pid_tgid); if (t == NULL) { - log_debug("[go-tls-close] failed getting conn tup from tls conn for pid %llu", pid_tgid >> 32); + log_debug("[go-tls-close] failed getting conn tup from tls conn for pid %llu", GET_USER_MODE_PID(pid_tgid)); return 0; } diff --git a/pkg/network/ebpf/c/shared-libraries/probes.h b/pkg/network/ebpf/c/shared-libraries/probes.h index a33e228a63b26..a45e9d0b494e1 100644 --- a/pkg/network/ebpf/c/shared-libraries/probes.h +++ b/pkg/network/ebpf/c/shared-libraries/probes.h @@ -4,6 +4,7 @@ #include "bpf_telemetry.h" #include "bpf_bypass.h" +#include "pid_tgid.h" #include "shared-libraries/types.h" static __always_inline void fill_path_safe(lib_path_t *path, const char *path_argument) { @@ -38,7 +39,7 @@ static __always_inline void do_sys_open_helper_enter(const char *filename) { } u64 pid_tgid = bpf_get_current_pid_tgid(); - path.pid = pid_tgid >> 32; + path.pid = GET_USER_MODE_PID(pid_tgid); bpf_map_update_with_telemetry(open_at_args, &pid_tgid, &path, BPF_ANY); return; } diff --git a/pkg/network/ebpf/c/sock.h b/pkg/network/ebpf/c/sock.h index c71e48a544f1d..997acc13edef6 100644 --- a/pkg/network/ebpf/c/sock.h +++ b/pkg/network/ebpf/c/sock.h @@ -8,6 +8,7 @@ #include "ip.h" #include "ipv6.h" #include "netns.h" +#include "pid_tgid.h" #ifdef COMPILE_CORE @@ -178,7 +179,7 @@ static __always_inline u16 _sk_family(struct sock *skp) { */ static __always_inline int read_conn_tuple_partial(conn_tuple_t* t, struct sock* skp, u64 pid_tgid, metadata_mask_t type) { int err = 0; - t->pid = pid_tgid >> 32; + t->pid = GET_USER_MODE_PID(pid_tgid); t->metadata = type; // Retrieve network namespace id first since addresses and ports may not be available for unconnected UDP diff --git a/pkg/network/ebpf/c/tracer.c b/pkg/network/ebpf/c/tracer.c index 8de25a2a37e6b..66cefbee1d08d 100644 --- a/pkg/network/ebpf/c/tracer.c +++ b/pkg/network/ebpf/c/tracer.c @@ -19,6 +19,7 @@ #include "tracer/port.h" #include "tracer/tcp_recv.h" #include "protocols/classification/protocol-classification.h" +#include "pid_tgid.h" __maybe_unused static __always_inline bool tcp_failed_connections_enabled() { __u64 val = 0; @@ -226,7 +227,7 @@ int BPF_BYPASSABLE_KPROBE(kprobe__tcp_done, struct sock *sk) { pid_ts_t *failed_conn_pid = bpf_map_lookup_elem(&tcp_ongoing_connect_pid, &skp_conn); if (failed_conn_pid) { bpf_map_delete_elem(&tcp_ongoing_connect_pid, &skp_conn); - t.pid = failed_conn_pid->pid_tgid >> 32; + t.pid = GET_USER_MODE_PID(failed_conn_pid->pid_tgid); } else { increment_telemetry_count(tcp_done_missing_pid); return 0; @@ -256,7 +257,7 @@ int BPF_BYPASSABLE_KPROBE(kprobe__tcp_close, struct sock *sk) { u64 pid_tgid = bpf_get_current_pid_tgid(); // Get network namespace id - log_debug("kprobe/tcp_close: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + log_debug("kprobe/tcp_close: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); if (!read_conn_tuple(&t, sk, pid_tgid, CONN_TYPE_TCP)) { return 0; } @@ -734,7 +735,7 @@ int BPF_BYPASSABLE_KRETPROBE(kretprobe__udpv6_recvmsg) { static __always_inline int handle_ret_udp_recvmsg_pre_4_7_0(int copied, void *udp_sock_map) { u64 pid_tgid = bpf_get_current_pid_tgid(); - log_debug("kretprobe/udp_recvmsg: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + log_debug("kretprobe/udp_recvmsg: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); // Retrieve socket pointer from kprobe via pid/tgid udp_recv_sock_t *st = bpf_map_lookup_elem(udp_sock_map, &pid_tgid); @@ -886,18 +887,18 @@ int BPF_BYPASSABLE_KPROBE(kprobe__tcp_retransmit_skb_pre_4_7_0, struct sock *sk) SEC("kretprobe/tcp_retransmit_skb") int BPF_BYPASSABLE_KRETPROBE(kretprobe__tcp_retransmit_skb, int ret) { - __u64 tid = bpf_get_current_pid_tgid(); + __u64 pid_tgid = bpf_get_current_pid_tgid(); if (ret < 0) { - bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid); + bpf_map_delete_elem(&pending_tcp_retransmit_skb, &pid_tgid); return 0; } - tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &tid); + tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &pid_tgid); if (args == NULL) { return 0; } struct sock *sk = args->sk; int segs = args->segs; - bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid); + bpf_map_delete_elem(&pending_tcp_retransmit_skb, &pid_tgid); log_debug("kretprobe/tcp_retransmit: segs: %d", segs); return handle_retransmit(sk, segs); } @@ -908,30 +909,30 @@ int BPF_BYPASSABLE_KRETPROBE(kretprobe__tcp_retransmit_skb, int ret) { SEC("kprobe/tcp_retransmit_skb") int BPF_BYPASSABLE_KPROBE(kprobe__tcp_retransmit_skb, struct sock *sk) { - u64 tid = bpf_get_current_pid_tgid(); + u64 pid_tgid = bpf_get_current_pid_tgid(); tcp_retransmit_skb_args_t args = {}; args.sk = sk; args.segs = 0; BPF_CORE_READ_INTO(&args.retrans_out_pre, tcp_sk(sk), retrans_out); - bpf_map_update_with_telemetry(pending_tcp_retransmit_skb, &tid, &args, BPF_ANY); + bpf_map_update_with_telemetry(pending_tcp_retransmit_skb, &pid_tgid, &args, BPF_ANY); return 0; } SEC("kretprobe/tcp_retransmit_skb") int BPF_BYPASSABLE_KRETPROBE(kretprobe__tcp_retransmit_skb, int rc) { log_debug("kretprobe/tcp_retransmit"); - u64 tid = bpf_get_current_pid_tgid(); + u64 pid_tgid = bpf_get_current_pid_tgid(); if (rc < 0) { - bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid); + bpf_map_delete_elem(&pending_tcp_retransmit_skb, &pid_tgid); return 0; } - tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &tid); + tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &pid_tgid); if (args == NULL) { return 0; } struct sock *sk = args->sk; u32 retrans_out_pre = args->retrans_out_pre; - bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid); + bpf_map_delete_elem(&pending_tcp_retransmit_skb, &pid_tgid); u32 retrans_out = 0; BPF_CORE_READ_INTO(&retrans_out, tcp_sk(sk), retrans_out); return handle_retransmit(sk, retrans_out - retrans_out_pre); @@ -942,7 +943,7 @@ int BPF_BYPASSABLE_KRETPROBE(kretprobe__tcp_retransmit_skb, int rc) { SEC("kprobe/tcp_connect") int BPF_BYPASSABLE_KPROBE(kprobe__tcp_connect, struct sock *skp) { u64 pid_tgid = bpf_get_current_pid_tgid(); - log_debug("kprobe/tcp_connect: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + log_debug("kprobe/tcp_connect: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); conn_tuple_t t = {}; if (!read_conn_tuple(&t, skp, 0, CONN_TYPE_TCP)) { @@ -971,8 +972,8 @@ int BPF_BYPASSABLE_KPROBE(kprobe__tcp_finish_connect, struct sock *skp) { } u64 pid_tgid = pid_tgid_p->pid_tgid; - t.pid = pid_tgid >> 32; - log_debug("kprobe/tcp_finish_connect: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + t.pid = GET_USER_MODE_PID(pid_tgid); + log_debug("kprobe/tcp_finish_connect: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); handle_tcp_stats(&t, skp, TCP_ESTABLISHED); handle_message(&t, 0, 0, CONN_DIRECTION_OUTGOING, 0, 0, PACKET_COUNT_NONE, skp); @@ -989,7 +990,7 @@ int BPF_BYPASSABLE_KRETPROBE(kretprobe__inet_csk_accept, struct sock *sk) { } u64 pid_tgid = bpf_get_current_pid_tgid(); - log_debug("kretprobe/inet_csk_accept: tgid: %llu, pid: %llu", pid_tgid >> 32, pid_tgid & 0xFFFFFFFF); + log_debug("kretprobe/inet_csk_accept: kernel thread id: %llu, user mode pid: %llu", GET_KERNEL_THREAD_ID(pid_tgid), GET_USER_MODE_PID(pid_tgid)); conn_tuple_t t = {}; if (!read_conn_tuple(&t, sk, pid_tgid, CONN_TYPE_TCP)) { diff --git a/pkg/network/ebpf/c/tracer/bind.h b/pkg/network/ebpf/c/tracer/bind.h index 67a01a2da4f4c..cc8c3dd99e5cc 100644 --- a/pkg/network/ebpf/c/tracer/bind.h +++ b/pkg/network/ebpf/c/tracer/bind.h @@ -1,6 +1,7 @@ #ifndef __TRACER_BIND_H #define __TRACER_BIND_H +#include "pid_tgid.h" #include "tracer/tracer.h" #include "tracer/maps.h" #include "tracer/port.h" @@ -19,7 +20,7 @@ static __always_inline u16 sockaddr_sin_port(struct sockaddr *addr) { } static __always_inline int sys_enter_bind(struct socket *sock, struct sockaddr *addr) { - __u64 tid = bpf_get_current_pid_tgid(); + __u64 pid_tgid = bpf_get_current_pid_tgid(); __u16 type = 0; bpf_probe_read_kernel_with_telemetry(&type, sizeof(__u16), &sock->type); @@ -28,7 +29,7 @@ static __always_inline int sys_enter_bind(struct socket *sock, struct sockaddr * } if (addr == NULL) { - log_debug("sys_enter_bind: could not read sockaddr, sock=%p, tid=%llu", sock, tid); + log_debug("sys_enter_bind: could not read sockaddr, sock=%p, pid_tgid=%llu", sock, pid_tgid); return 0; } @@ -49,19 +50,19 @@ static __always_inline int sys_enter_bind(struct socket *sock, struct sockaddr * args.addr = addr; - bpf_map_update_with_telemetry(pending_bind, &tid, &args, BPF_ANY); - log_debug("sys_enter_bind: started a bind on UDP sock=%p tid=%llu", sock, tid); + bpf_map_update_with_telemetry(pending_bind, &pid_tgid, &args, BPF_ANY); + log_debug("sys_enter_bind: started a bind on UDP sock=%p pid_tgid=%llu", sock, pid_tgid); return 0; } static __always_inline int sys_exit_bind(__s64 ret) { - __u64 tid = bpf_get_current_pid_tgid(); + __u64 pid_tgid = bpf_get_current_pid_tgid(); // bail if this bind() is not the one we're instrumenting - bind_syscall_args_t *args = bpf_map_lookup_elem(&pending_bind, &tid); + bind_syscall_args_t *args = bpf_map_lookup_elem(&pending_bind, &pid_tgid); - log_debug("sys_exit_bind: tid=%llu, ret=%lld", tid, ret); + log_debug("sys_exit_bind: pid_tgid=%llu, ret=%lld", pid_tgid, ret); if (args == NULL) { log_debug("sys_exit_bind: was not a UDP bind, will not process"); @@ -70,7 +71,7 @@ static __always_inline int sys_exit_bind(__s64 ret) { struct sock * sk = args->sk; struct sockaddr *addr = args->addr; - bpf_map_delete_elem(&pending_bind, &tid); + bpf_map_delete_elem(&pending_bind, &pid_tgid); if (ret != 0) { return 0; diff --git a/pkg/network/ebpf/c/tracer/stats.h b/pkg/network/ebpf/c/tracer/stats.h index b05811c86190a..862d8e32f81f1 100644 --- a/pkg/network/ebpf/c/tracer/stats.h +++ b/pkg/network/ebpf/c/tracer/stats.h @@ -16,6 +16,7 @@ #include "protocols/tls/tags-types.h" #include "ip.h" #include "skb.h" +#include "pid_tgid.h" #ifdef COMPILE_PREBUILT static __always_inline __u64 offset_rtt(); @@ -286,7 +287,7 @@ static __always_inline int handle_skb_consume_udp(struct sock *sk, struct sk_buf flip_tuple(&t); log_debug("skb_consume_udp: bytes=%d", data_len); - t.pid = pid_tgid >> 32; + t.pid = GET_USER_MODE_PID(pid_tgid); t.netns = get_netns_from_sock(sk); return handle_message(&t, 0, data_len, CONN_DIRECTION_UNKNOWN, 0, 1, PACKET_COUNT_INCREMENT, sk); } diff --git a/pkg/network/tracer/connection/ebpfless/map_utils.go b/pkg/network/tracer/connection/ebpfless/map_utils.go new file mode 100644 index 0000000000000..3e41f4832707d --- /dev/null +++ b/pkg/network/tracer/connection/ebpfless/map_utils.go @@ -0,0 +1,19 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +package ebpfless + +// WriteMapWithSizeLimit updates a map via m[key] = val. +// However, if the map would overflow sizeLimit, it returns false instead. +func WriteMapWithSizeLimit[Key comparable, Val any](m map[Key]Val, key Key, val Val, sizeLimit int) bool { + _, exists := m[key] + if !exists && len(m) >= sizeLimit { + return false + } + m[key] = val + return true +} diff --git a/pkg/network/tracer/connection/ebpfless/map_utils_test.go b/pkg/network/tracer/connection/ebpfless/map_utils_test.go new file mode 100644 index 0000000000000..e387328d00610 --- /dev/null +++ b/pkg/network/tracer/connection/ebpfless/map_utils_test.go @@ -0,0 +1,38 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +package ebpfless + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestWriteMapWithSizeLimit(t *testing.T) { + m := map[string]int{} + + // not full: any write should work + ok := WriteMapWithSizeLimit(m, "foo", 123, 1) + require.True(t, ok) + + expectedFoo := map[string]int{ + "foo": 123, + } + require.Equal(t, expectedFoo, m) + + // full: shouldn't write a new key + ok = WriteMapWithSizeLimit(m, "bar", 456, 1) + require.False(t, ok) + require.Equal(t, expectedFoo, m) + + // full: replacing key should still work + ok = WriteMapWithSizeLimit(m, "foo", 789, 1) + require.True(t, ok) + require.Equal(t, map[string]int{ + "foo": 789, + }, m) +} diff --git a/pkg/network/tracer/connection/ebpfless/tcp_processor.go b/pkg/network/tracer/connection/ebpfless/tcp_processor.go index b312f460269ee..73ea6a1152aa3 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_processor.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_processor.go @@ -9,6 +9,7 @@ package ebpfless import ( "fmt" + "github.com/DataDog/datadog-agent/pkg/util/log" "syscall" "time" @@ -17,7 +18,7 @@ import ( "github.com/google/gopacket/layers" "github.com/DataDog/datadog-agent/pkg/network" - "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/network/config" ) type connectionState struct { @@ -51,18 +52,40 @@ type connectionState struct { // remoteFinSeq is the tcp.Seq number for the incoming FIN (including any payload length) remoteFinSeq uint32 + // rttTracker is used to track round trip times rttTracker rttTracker + + // lastUpdateEpoch contains the last timestamp this connection sent/received a packet + // TODO find a way to combine this with ConnectionStats.lastUpdateEpoch + // This exists because connections in pendingConns don't have a ConnectionStats object yet. + // Can we make all connections in TCPProcessor have a ConnectionStats no matter what, and + // filter them out in GetConnections? + lastUpdateEpoch uint64 +} + +func (st *connectionState) hasMissedHandshake() bool { + return st.localSynState == synStateMissed || st.remoteSynState == synStateMissed } // TCPProcessor encapsulates TCP state tracking for the ebpfless tracer type TCPProcessor struct { - conns map[network.ConnectionTuple]connectionState + cfg *config.Config + // pendingConns contains connections with tcpState == connStatAttempted + pendingConns map[network.ConnectionTuple]*connectionState + // establishedConns contains connections with tcpState == connStatEstablished + establishedConns map[network.ConnectionTuple]*connectionState } +// TODO make this into a config value +const maxPendingConns = 4096 +const pendingConnTimeoutNs = uint64(5 * time.Second) + // NewTCPProcessor constructs an empty TCPProcessor -func NewTCPProcessor() *TCPProcessor { +func NewTCPProcessor(cfg *config.Config) *TCPProcessor { return &TCPProcessor{ - conns: map[network.ConnectionTuple]connectionState{}, + cfg: cfg, + pendingConns: make(map[network.ConnectionTuple]*connectionState, maxPendingConns), + establishedConns: make(map[network.ConnectionTuple]*connectionState, cfg.MaxTrackedConnections), } } @@ -125,9 +148,13 @@ func (t *TCPProcessor) updateSynFlag(conn *network.ConnectionStats, st *connecti updateConnStatsForOpen(conn) } // if both synStates are ack'd, move to established - if st.tcpState == connStatAttempted && st.localSynState == synStateAcked && st.remoteSynState == synStateAcked { + if st.tcpState == connStatAttempted && st.localSynState.isSynAcked() && st.remoteSynState.isSynAcked() { st.tcpState = connStatEstablished - conn.Monotonic.TCPEstablished++ + if st.hasMissedHandshake() { + statsTelemetry.missedTCPHandshakes.Inc() + } else { + conn.Monotonic.TCPEstablished++ + } } } @@ -136,6 +163,7 @@ func (t *TCPProcessor) updateSynFlag(conn *network.ConnectionStats, st *connecti func (t *TCPProcessor) updateTCPStats(conn *network.ConnectionStats, st *connectionState, pktType uint8, tcp *layers.TCP, payloadLen uint16, timestampNs uint64) { nextSeq := calcNextSeq(tcp, payloadLen) + st.lastUpdateEpoch = timestampNs if pktType == unix.PACKET_OUTGOING { conn.Monotonic.SentPackets++ // packetCanRetransmit filters out packets that look like retransmits but aren't, like TCP keepalives @@ -155,7 +183,7 @@ func (t *TCPProcessor) updateTCPStats(conn *network.ConnectionStats, st *connect ackOutdated := !st.hasLocalAck || isSeqBefore(st.lastLocalAck, tcp.Ack) if tcp.ACK && ackOutdated { // wait until data comes in via synStateAcked - if st.hasLocalAck && st.remoteSynState == synStateAcked { + if st.hasLocalAck && st.remoteSynState.isSynAcked() { ackDiff := tcp.Ack - st.lastLocalAck isFinAck := st.hasRemoteFin && tcp.Ack == st.remoteFinSeq if isFinAck { @@ -232,13 +260,13 @@ func (t *TCPProcessor) updateRstFlag(conn *network.ConnectionStats, st *connecti // Process handles a TCP packet, calculating stats and keeping track of its state according to the // TCP state machine. -func (t *TCPProcessor) Process(conn *network.ConnectionStats, timestampNs uint64, pktType uint8, ip4 *layers.IPv4, ip6 *layers.IPv6, tcp *layers.TCP) error { +func (t *TCPProcessor) Process(conn *network.ConnectionStats, timestampNs uint64, pktType uint8, ip4 *layers.IPv4, ip6 *layers.IPv6, tcp *layers.TCP) (ProcessResult, error) { if pktType != unix.PACKET_OUTGOING && pktType != unix.PACKET_HOST { - return fmt.Errorf("TCPProcessor saw invalid pktType: %d", pktType) + return ProcessResultNone, fmt.Errorf("TCPProcessor saw invalid pktType: %d", pktType) } payloadLen, err := TCPPayloadLen(conn.Family, ip4, ip6, tcp) if err != nil { - return err + return ProcessResultNone, err } log.TraceFunc(func() string { @@ -247,16 +275,93 @@ func (t *TCPProcessor) Process(conn *network.ConnectionStats, timestampNs uint64 // skip invalid packets we don't recognize: if checkInvalidTCP(tcp) { - return nil + return ProcessResultNone, nil } - st := t.conns[conn.ConnectionTuple] + st := t.getConn(conn.ConnectionTuple) + origState := st.tcpState - t.updateSynFlag(conn, &st, pktType, tcp, payloadLen) - t.updateTCPStats(conn, &st, pktType, tcp, payloadLen, timestampNs) - t.updateFinFlag(conn, &st, pktType, tcp, payloadLen) - t.updateRstFlag(conn, &st, pktType, tcp, payloadLen) + t.updateSynFlag(conn, st, pktType, tcp, payloadLen) + t.updateTCPStats(conn, st, pktType, tcp, payloadLen, timestampNs) + t.updateFinFlag(conn, st, pktType, tcp, payloadLen) + t.updateRstFlag(conn, st, pktType, tcp, payloadLen) + + stateChanged := st.tcpState != origState + if stateChanged { + ok := t.moveConn(conn.ConnectionTuple, st) + // if the map is full then we are unable to move the connection, report that + if !ok { + return ProcessResultMapFull, nil + } + } + + // if the connection is still established, we should update the connection map + if st.tcpState == connStatEstablished { + return ProcessResultStoreConn, nil + } + // if the connection just closed, store it in the tracer's closeCallback + if st.tcpState == connStatClosed && stateChanged { + return ProcessResultCloseConn, nil + } + return ProcessResultNone, nil +} - t.conns[conn.ConnectionTuple] = st - return nil +func (t *TCPProcessor) getConn(tuple network.ConnectionTuple) *connectionState { + if st, ok := t.establishedConns[tuple]; ok { + return st + } + if st, ok := t.pendingConns[tuple]; ok { + return st + } + // otherwise, create a fresh state object that will be stored by moveConn later + return &connectionState{} +} + +// RemoveConn clears a ConnectionTuple from its internal state. +func (t *TCPProcessor) RemoveConn(tuple network.ConnectionTuple) { + delete(t.pendingConns, tuple) + delete(t.establishedConns, tuple) +} + +// moveConn moves a connection to the correct map based on its tcpState. +// If it had to drop the connection because the target map was full, it returns false. +func (t *TCPProcessor) moveConn(tuple network.ConnectionTuple, st *connectionState) bool { + t.RemoveConn(tuple) + + switch st.tcpState { + // For this case, simply let closed connections disappear. Process() will return + // ProcessResultCloseConn letting the ebpfless tracer know the connection has closed. + case connStatClosed: + case connStatAttempted: + ok := WriteMapWithSizeLimit(t.pendingConns, tuple, st, maxPendingConns) + if !ok { + statsTelemetry.droppedPendingConns.Inc() + } + return ok + case connStatEstablished: + maxTrackedConns := int(t.cfg.MaxTrackedConnections) + ok := WriteMapWithSizeLimit(t.establishedConns, tuple, st, maxTrackedConns) + if !ok { + statsTelemetry.droppedEstablishedConns.Inc() + } + return ok + } + return true +} + +// CleanupExpiredPendingConns iterates through pendingConns and removes those that +// have existed too long - in normal TCP, they should become established right away. +// +// This is only required for pendingConns because the tracer already has logic to remove +// established connections (connections that have ConnectionStats) +func (t *TCPProcessor) CleanupExpiredPendingConns(timestampNs uint64) { + for tuple, st := range t.pendingConns { + timeoutTime := st.lastUpdateEpoch + pendingConnTimeoutNs + + if timeoutTime <= timestampNs { + delete(t.pendingConns, tuple) + + statsTelemetry.expiredPendingConns.Inc() + } + } } diff --git a/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go b/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go index 8656a73463d36..b7d525bce1531 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go @@ -8,9 +8,11 @@ package ebpfless import ( + "github.com/DataDog/datadog-agent/pkg/network/config" "net" "syscall" "testing" + "time" "golang.org/x/sys/unix" @@ -166,22 +168,24 @@ func (pb packetBuilder) outgoing(payloadLen uint16, relSeq, relAck uint32, flags } func newTCPTestFixture(t *testing.T) *tcpTestFixture { + cfg := config.New() return &tcpTestFixture{ t: t, - tcp: NewTCPProcessor(), + tcp: NewTCPProcessor(cfg), conn: nil, } } -func (fixture *tcpTestFixture) runPkt(pkt testCapture) { +func (fixture *tcpTestFixture) runPkt(pkt testCapture) ProcessResult { if fixture.conn == nil { fixture.conn = makeTCPStates(pkt) } - err := fixture.tcp.Process(fixture.conn, pkt.timestampNs, pkt.pktType, pkt.ipv4, pkt.ipv6, pkt.tcp) + result, err := fixture.tcp.Process(fixture.conn, pkt.timestampNs, pkt.pktType, pkt.ipv4, pkt.ipv6, pkt.tcp) require.NoError(fixture.t, err) + return result } -func (fixture *tcpTestFixture) runPkts(packets []testCapture) { //nolint:unused // TODO +func (fixture *tcpTestFixture) runPkts(packets []testCapture) { for _, pkt := range packets { fixture.runPkt(pkt) } @@ -197,7 +201,7 @@ func (fixture *tcpTestFixture) runAgainstState(packets []testCapture, expected [ fixture.runPkt(pkt) connTuple := fixture.conn.ConnectionTuple - actual := fixture.tcp.conns[connTuple].tcpState + actual := fixture.tcp.getConn(connTuple).tcpState actualStrs = append(actualStrs, labelForState(actual)) } require.Equal(fixture.t, expectedStrs, actualStrs) @@ -255,6 +259,9 @@ func testBasicHandshake(t *testing.T, pb packetBuilder) { } require.Equal(t, expectedStats, f.conn.Monotonic) + + require.Empty(t, f.tcp.pendingConns) + require.Empty(t, f.tcp.establishedConns) } var lowerSeq uint32 = 2134452051 @@ -322,6 +329,9 @@ func testReversedBasicHandshake(t *testing.T, pb packetBuilder) { TCPClosed: 1, } require.Equal(t, expectedStats, f.conn.Monotonic) + + require.Empty(t, f.tcp.pendingConns) + require.Empty(t, f.tcp.establishedConns) } func TestReversedBasicHandshake(t *testing.T) { @@ -614,9 +624,8 @@ func TestConnReset(t *testing.T) { require.Equal(t, expectedStats, f.conn.Monotonic) } -func TestConnectTwice(t *testing.T) { - // same as TestImmediateFin but everything happens twice - +func TestProcessResult(t *testing.T) { + // same as TestImmediateFin but checks ProcessResult pb := newPacketBuilder(lowerSeq, higherSeq) basicHandshake := []testCapture{ pb.incoming(0, 0, 0, SYN), @@ -628,40 +637,20 @@ func TestConnectTwice(t *testing.T) { pb.outgoing(0, 2, 2, ACK), } - expectedClientStates := []connStatus{ - connStatAttempted, - connStatAttempted, - connStatEstablished, - // active close begins here - connStatEstablished, - connStatEstablished, - connStatClosed, + processResults := []ProcessResult{ + ProcessResultNone, + ProcessResultNone, + ProcessResultStoreConn, + ProcessResultStoreConn, + ProcessResultStoreConn, + ProcessResultCloseConn, } f := newTCPTestFixture(t) - f.runAgainstState(basicHandshake, expectedClientStates) - state := f.tcp.conns[f.conn.ConnectionTuple] - // make sure the TCP state was erased after the connection was closed - require.Equal(t, connectionState{ - tcpState: connStatClosed, - }, state) - - // second connection here - f.runAgainstState(basicHandshake, expectedClientStates) - - require.Empty(t, f.conn.TCPFailures) - - expectedStats := network.StatCounters{ - SentBytes: 0, - RecvBytes: 0, - SentPackets: 3 * 2, - RecvPackets: 3 * 2, - Retransmits: 0, - TCPEstablished: 1 * 2, - TCPClosed: 1 * 2, + for i, pkt := range basicHandshake { + require.Equal(t, processResults[i], f.runPkt(pkt), "packet #%d has the wrong ProcessResult", i) } - require.Equal(t, expectedStats, f.conn.Monotonic) } func TestSimultaneousClose(t *testing.T) { @@ -775,3 +764,54 @@ func TestOpenCloseConn(t *testing.T) { f.runPkt(pb.incoming(0, 0, 0, SYN)) require.False(t, f.conn.IsClosed) } +func TestPreexistingConn(t *testing.T) { + pb := newPacketBuilder(lowerSeq, higherSeq) + + f := newTCPTestFixture(t) + + capture := []testCapture{ + // just sending data, no SYN + pb.outgoing(1, 10, 10, ACK), + pb.incoming(1, 10, 11, ACK), + // active close after sending no data + pb.outgoing(0, 11, 11, FIN|ACK), + pb.incoming(0, 11, 12, FIN|ACK), + pb.outgoing(0, 12, 12, ACK), + } + f.runPkts(capture) + + require.Empty(t, f.conn.TCPFailures) + + expectedStats := network.StatCounters{ + SentBytes: 1, + RecvBytes: 1, + SentPackets: 3, + RecvPackets: 2, + Retransmits: 0, + TCPEstablished: 0, // we missed when it established + TCPClosed: 1, + } + require.Equal(t, expectedStats, f.conn.Monotonic) +} + +func TestPendingConnExpiry(t *testing.T) { + now := uint64(time.Now().UnixNano()) + + pb := newPacketBuilder(lowerSeq, higherSeq) + pkt := pb.outgoing(0, 0, 0, SYN) + pkt.timestampNs = now + + f := newTCPTestFixture(t) + + f.runPkt(pkt) + require.Len(t, f.tcp.pendingConns, 1) + + // if no time has passed, should not remove the connection + f.tcp.CleanupExpiredPendingConns(now) + require.Len(t, f.tcp.pendingConns, 1) + + // if too much time has passed, should remove the connection + tenSecNs := uint64((10 * time.Second).Nanoseconds()) + f.tcp.CleanupExpiredPendingConns(now + tenSecNs) + require.Empty(t, f.tcp.pendingConns) +} diff --git a/pkg/network/tracer/connection/ebpfless/tcp_utils.go b/pkg/network/tracer/connection/ebpfless/tcp_utils.go index 1969fd85a5bc1..7a30737e734ae 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_utils.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_utils.go @@ -21,18 +21,42 @@ import ( const ebpflessModuleName = "ebpfless_network_tracer" +// ProcessResult represents what the ebpfless tracer should do with ConnectionStats after processing a packet +type ProcessResult uint8 + +const ( + // ProcessResultNone - the updated ConnectionStats should NOT be stored in the connection map. + // Usually, this is because the connection is not established yet. + ProcessResultNone ProcessResult = iota + // ProcessResultStoreConn - the updated ConnectionStats should be stored in the connection map. + // This happens when the connection is established. + ProcessResultStoreConn + // ProcessResultCloseConn - this connection is done and its ConnectionStats should be passed + // to the ebpfless tracer's closed connection handler. + ProcessResultCloseConn + // ProcessResultMapFull - this connection can't be tracked because the TCPProcessor's connection + // map is full. This connection should be removed from the tracer as well. + ProcessResultMapFull +) + var statsTelemetry = struct { - missedTCPConnections telemetry.Counter - missingTCPFlags telemetry.Counter - tcpSynAndFin telemetry.Counter - tcpRstAndSyn telemetry.Counter - tcpRstAndFin telemetry.Counter + expiredPendingConns telemetry.Counter + droppedPendingConns telemetry.Counter + droppedEstablishedConns telemetry.Counter + missedTCPHandshakes telemetry.Counter + missingTCPFlags telemetry.Counter + tcpSynAndFin telemetry.Counter + tcpRstAndSyn telemetry.Counter + tcpRstAndFin telemetry.Counter }{ - telemetry.NewCounter(ebpflessModuleName, "missed_tcp_connections", []string{}, "Counter measuring the number of TCP connections where we missed the SYN handshake"), - telemetry.NewCounter(ebpflessModuleName, "missing_tcp_flags", []string{}, "Counter measuring packets encountered with none of SYN, FIN, ACK, RST set"), - telemetry.NewCounter(ebpflessModuleName, "tcp_syn_and_fin", []string{}, "Counter measuring packets encountered with SYN+FIN together"), - telemetry.NewCounter(ebpflessModuleName, "tcp_rst_and_syn", []string{}, "Counter measuring packets encountered with RST+SYN together"), - telemetry.NewCounter(ebpflessModuleName, "tcp_rst_and_fin", []string{}, "Counter measuring packets encountered with RST+FIN together"), + expiredPendingConns: telemetry.NewCounter(ebpflessModuleName, "expired_pending_conns", nil, "Counter measuring the number of TCP connections which expired because it took too long to complete the handshake"), + droppedPendingConns: telemetry.NewCounter(ebpflessModuleName, "dropped_pending_conns", nil, "Counter measuring the number of TCP connections which were dropped during the handshake (because the map was full)"), + droppedEstablishedConns: telemetry.NewCounter(ebpflessModuleName, "dropped_established_conns", nil, "Counter measuring the number of TCP connections which were dropped while established (because the map was full)"), + missedTCPHandshakes: telemetry.NewCounter(ebpflessModuleName, "missed_tcp_handshakes", nil, "Counter measuring the number of TCP connections where we missed the SYN handshake"), + missingTCPFlags: telemetry.NewCounter(ebpflessModuleName, "missing_tcp_flags", nil, "Counter measuring packets encountered with none of SYN, FIN, ACK, RST set"), + tcpSynAndFin: telemetry.NewCounter(ebpflessModuleName, "tcp_syn_and_fin", nil, "Counter measuring packets encountered with SYN+FIN together"), + tcpRstAndSyn: telemetry.NewCounter(ebpflessModuleName, "tcp_rst_and_syn", nil, "Counter measuring packets encountered with RST+SYN together"), + tcpRstAndFin: telemetry.NewCounter(ebpflessModuleName, "tcp_rst_and_fin", nil, "Counter measuring packets encountered with RST+FIN together"), } const tcpSeqMidpoint = 0x80000000 @@ -54,9 +78,16 @@ var connStatusLabels = []string{ type synState uint8 const ( + // synStateNone - Nothing seen yet (initial state) synStateNone synState = iota + // synStateSent - We have seen the SYN but not its ACK synStateSent + // synStateAcked - SYN is ACK'd for this side of the connection. + // If both sides are synStateAcked, the connection is established. synStateAcked + // synStateMissed is effectively the same as synStateAcked but represents + // capturing a preexisting connection where we didn't get to see the SYN. + synStateMissed ) func (ss *synState) update(synFlag, ackFlag bool) { @@ -68,13 +99,20 @@ func (ss *synState) update(synFlag, ackFlag bool) { if *ss == synStateSent && ackFlag { *ss = synStateAcked } + + // this allows synStateMissed to recover via SYN in order to pass TestUnusualAckSyn + if *ss == synStateMissed && synFlag { + *ss = synStateAcked + } // if we see ACK'd traffic but missed the SYN, assume the connection started before // the datadog-agent starts. if *ss == synStateNone && ackFlag { - statsTelemetry.missedTCPConnections.Inc() - *ss = synStateAcked + *ss = synStateMissed } } +func (ss *synState) isSynAcked() bool { + return *ss == synStateAcked || *ss == synStateMissed +} func labelForState(tcpState connStatus) string { idx := int(tcpState) diff --git a/pkg/network/tracer/connection/ebpfless_tracer.go b/pkg/network/tracer/connection/ebpfless_tracer.go index 8bb0a54f170ec..3eb5a03344b2a 100644 --- a/pkg/network/tracer/connection/ebpfless_tracer.go +++ b/pkg/network/tracer/connection/ebpfless_tracer.go @@ -40,9 +40,11 @@ const ( var ( ebpfLessTracerTelemetry = struct { - skippedPackets telemetry.Counter + skippedPackets telemetry.Counter + droppedConnections telemetry.Counter }{ telemetry.NewCounter(ebpfLessTelemetryPrefix, "skipped_packets", []string{"reason"}, "Counter measuring skipped packets"), + telemetry.NewCounter(ebpfLessTelemetryPrefix, "dropped_connections", nil, "Counter measuring dropped connections"), } ) @@ -81,7 +83,7 @@ func newEbpfLessTracer(cfg *config.Config) (*ebpfLessTracer, error) { exit: make(chan struct{}), scratchConn: &network.ConnectionStats{}, udp: &udpProcessor{}, - tcp: ebpfless.NewTCPProcessor(), + tcp: ebpfless.NewTCPProcessor(cfg), conns: make(map[network.ConnectionTuple]*network.ConnectionStats, cfg.MaxTrackedConnections), boundPorts: ebpfless.NewBoundPorts(cfg), cookieHasher: newCookieHasher(), @@ -96,7 +98,7 @@ func newEbpfLessTracer(cfg *config.Config) (*ebpfLessTracer, error) { } // Start begins collecting network connection data. -func (t *ebpfLessTracer) Start(_ func(*network.ConnectionStats)) error { +func (t *ebpfLessTracer) Start(closeCallback func(*network.ConnectionStats)) error { if err := t.boundPorts.Start(); err != nil { return fmt.Errorf("could not update bound ports: %w", err) } @@ -123,7 +125,7 @@ func (t *ebpfLessTracer) Start(_ func(*network.ConnectionStats)) error { return nil } - if err := t.processConnection(pktType, &ip4, &ip6, &udp, &tcp, decoded); err != nil { + if err := t.processConnection(pktType, &ip4, &ip6, &udp, &tcp, decoded, closeCallback); err != nil { log.Warnf("could not process packet: %s", err) } @@ -147,21 +149,24 @@ func (t *ebpfLessTracer) processConnection( udp *layers.UDP, tcp *layers.TCP, decoded []gopacket.LayerType, + closeCallback func(*network.ConnectionStats), ) error { t.scratchConn.Source, t.scratchConn.Dest = util.Address{}, util.Address{} t.scratchConn.SPort, t.scratchConn.DPort = 0, 0 t.scratchConn.TCPFailures = make(map[uint16]uint32) - var udpPresent, tcpPresent bool + var ip4Present, ip6Present, udpPresent, tcpPresent bool for _, layerType := range decoded { switch layerType { case layers.LayerTypeIPv4: t.scratchConn.Source = util.AddressFromNetIP(ip4.SrcIP) t.scratchConn.Dest = util.AddressFromNetIP(ip4.DstIP) t.scratchConn.Family = network.AFINET + ip4Present = true case layers.LayerTypeIPv6: t.scratchConn.Source = util.AddressFromNetIP(ip6.SrcIP) t.scratchConn.Dest = util.AddressFromNetIP(ip6.DstIP) t.scratchConn.Family = network.AFINET6 + ip6Present = true case layers.LayerTypeTCP: t.scratchConn.SPort = uint16(tcp.SrcPort) t.scratchConn.DPort = uint16(tcp.DstPort) @@ -175,15 +180,15 @@ func (t *ebpfLessTracer) processConnection( } } - // check if have all the basic pieces + // check if we have all the basic pieces if !udpPresent && !tcpPresent { log.Debugf("ignoring packet since its not udp or tcp") ebpfLessTracerTelemetry.skippedPackets.Inc("not_tcp_udp") return nil } - flipSourceDest(t.scratchConn, pktType) t.determineConnectionDirection(t.scratchConn, pktType) + flipSourceDest(t.scratchConn, pktType) t.m.Lock() defer t.m.Unlock() @@ -201,21 +206,25 @@ func (t *ebpfLessTracer) processConnection( if ts, err = ddebpf.NowNanoseconds(); err != nil { return fmt.Errorf("error getting last updated timestamp for connection: %w", err) } + conn.LastUpdateEpoch = uint64(ts) - if ip4 == nil && ip6 == nil { + if !ip4Present && !ip6Present { return nil } + + var result ebpfless.ProcessResult switch conn.Type { case network.UDP: - if (ip4 != nil && !t.config.CollectUDPv4Conns) || (ip6 != nil && !t.config.CollectUDPv6Conns) { + if (ip4Present && !t.config.CollectUDPv4Conns) || (ip6Present && !t.config.CollectUDPv6Conns) { return nil } + result = ebpfless.ProcessResultStoreConn err = t.udp.process(conn, pktType, udp) case network.TCP: - if (ip4 != nil && !t.config.CollectTCPv4Conns) || (ip6 != nil && !t.config.CollectTCPv6Conns) { + if (ip4Present && !t.config.CollectTCPv4Conns) || (ip6Present && !t.config.CollectTCPv6Conns) { return nil } - err = t.tcp.Process(conn, uint64(ts), pktType, ip4, ip6, tcp) + result, err = t.tcp.Process(conn, uint64(ts), pktType, ip4, ip6, tcp) default: err = fmt.Errorf("unsupported connection type %d", conn.Type) } @@ -224,14 +233,30 @@ func (t *ebpfLessTracer) processConnection( return fmt.Errorf("error processing connection: %w", err) } - if conn.Type == network.UDP || conn.Monotonic.TCPEstablished > 0 { - conn.LastUpdateEpoch = uint64(ts) - t.conns[t.scratchConn.ConnectionTuple] = conn - } - log.TraceFunc(func() string { return fmt.Sprintf("connection: %s", conn) }) + + switch result { + case ebpfless.ProcessResultNone: + case ebpfless.ProcessResultStoreConn: + maxTrackedConns := int(t.config.MaxTrackedConnections) + ok := ebpfless.WriteMapWithSizeLimit(t.conns, conn.ConnectionTuple, conn, maxTrackedConns) + if !ok { + // we don't have enough space to add this connection, remove its TCP state tracking + if conn.Type == network.TCP { + t.tcp.RemoveConn(conn.ConnectionTuple) + } + ebpfLessTracerTelemetry.droppedConnections.Inc() + } + case ebpfless.ProcessResultCloseConn: + delete(t.conns, conn.ConnectionTuple) + closeCallback(conn) + case ebpfless.ProcessResultMapFull: + delete(t.conns, conn.ConnectionTuple) + ebpfLessTracerTelemetry.droppedConnections.Inc() + } + return nil } @@ -278,6 +303,12 @@ func (t *ebpfLessTracer) GetConnections(buffer *network.ConnectionBuffer, filter t.m.Lock() defer t.m.Unlock() + // use GetConnections to periodically cleanup pending connections + err := t.cleanupPendingConns() + if err != nil { + return err + } + if len(t.conns) == 0 { return nil } @@ -296,17 +327,35 @@ func (t *ebpfLessTracer) GetConnections(buffer *network.ConnectionBuffer, filter return nil } +// cleanupPendingConns removes pending connections from the TCP tracer. +// For more information, refer to CleanupExpiredPendingConns +func (t *ebpfLessTracer) cleanupPendingConns() error { + ts, err := ddebpf.NowNanoseconds() + if err != nil { + return fmt.Errorf("error getting last updated timestamp for connection: %w", err) + } + t.tcp.CleanupExpiredPendingConns(uint64(ts)) + return nil +} + // FlushPending forces any closed connections waiting for batching to be processed immediately. func (t *ebpfLessTracer) FlushPending() {} +func (t *ebpfLessTracer) remove(conn *network.ConnectionStats) error { + delete(t.conns, conn.ConnectionTuple) + if conn.Type == network.TCP { + t.tcp.RemoveConn(conn.ConnectionTuple) + } + return nil +} + // Remove deletes the connection from tracking state. // It does not prevent the connection from re-appearing later, if additional traffic occurs. func (t *ebpfLessTracer) Remove(conn *network.ConnectionStats) error { t.m.Lock() defer t.m.Unlock() - delete(t.conns, conn.ConnectionTuple) - return nil + return t.remove(conn) } // GetMap returns the underlying named map. This is useful if any maps are shared with other eBPF components. diff --git a/pkg/network/tracer/tracer_linux_test.go b/pkg/network/tracer/tracer_linux_test.go index d90928d560da8..46f33ac7fa405 100644 --- a/pkg/network/tracer/tracer_linux_test.go +++ b/pkg/network/tracer/tracer_linux_test.go @@ -109,7 +109,7 @@ func (s *TracerSuite) TestTCPRemoveEntries() { defer c2.Close() assert.EventuallyWithT(t, func(ct *assert.CollectT) { - conn, ok := findConnection(c2.LocalAddr(), c2.RemoteAddr(), getConnections(t, tr)) + conn, ok := findConnection(c2.LocalAddr(), c2.RemoteAddr(), getConnections(ct, tr)) if !assert.True(ct, ok) { return } @@ -123,9 +123,9 @@ func (s *TracerSuite) TestTCPRemoveEntries() { }, 3*time.Second, 100*time.Millisecond) // Make sure the first connection got cleaned up - assert.Eventually(t, func() bool { - _, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(t, tr)) - return !ok + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + _, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(ct, tr)) + require.False(ct, ok) }, 5*time.Second, 100*time.Millisecond) } @@ -735,10 +735,10 @@ func (s *TracerSuite) TestGatewayLookupEnabled() { dnsServerAddr := &net.UDPAddr{IP: dnsAddr, Port: 53} var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond) require.NotNil(t, conn.Via, "connection is missing via: %s", conn) @@ -791,10 +791,10 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { dnsClientAddr := &net.UDPAddr{IP: net.ParseIP(clientIP), Port: clientPort} dnsServerAddr := &net.UDPAddr{IP: destAddr, Port: 53} var c *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond, "connection not found") require.Nil(t, c.Via) @@ -804,10 +804,10 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { }, 6*time.Second, 100*time.Millisecond, "failed to send dns query") dnsClientAddr = &net.UDPAddr{IP: net.ParseIP(clientIP), Port: clientPort} - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond, "connection not found") require.Nil(t, c.Via) @@ -1529,25 +1529,25 @@ func testUDPReusePort(t *testing.T, udpnet string, ip string) { assert.EventuallyWithT(t, func(ct *assert.CollectT) { //nolint:revive // TODO // use t instead of ct because getConnections uses require (not assert), and we get a better error message that way - connections := getConnections(t, tr) + connections := getConnections(ct, tr) incoming, ok := findConnection(c.RemoteAddr(), c.LocalAddr(), connections) - if assert.True(t, ok, "unable to find incoming connection") { - assert.Equal(t, network.INCOMING, incoming.Direction) + if assert.True(ct, ok, "unable to find incoming connection") { + assert.Equal(ct, network.INCOMING, incoming.Direction) // make sure the inverse values are seen for the other message - assert.Equal(t, serverMessageSize, int(incoming.Monotonic.SentBytes), "incoming sent") - assert.Equal(t, clientMessageSize, int(incoming.Monotonic.RecvBytes), "incoming recv") - assert.True(t, incoming.IntraHost, "incoming intrahost") + assert.Equal(ct, serverMessageSize, int(incoming.Monotonic.SentBytes), "incoming sent") + assert.Equal(ct, clientMessageSize, int(incoming.Monotonic.RecvBytes), "incoming recv") + assert.True(ct, incoming.IntraHost, "incoming intrahost") } outgoing, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), connections) - if assert.True(t, ok, "unable to find outgoing connection") { - assert.Equal(t, network.OUTGOING, outgoing.Direction) + if assert.True(ct, ok, "unable to find outgoing connection") { + assert.Equal(ct, network.OUTGOING, outgoing.Direction) - assert.Equal(t, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") - assert.Equal(t, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") - assert.True(t, outgoing.IntraHost, "outgoing intrahost") + assert.Equal(ct, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") + assert.Equal(ct, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") + assert.True(ct, outgoing.IntraHost, "outgoing intrahost") } }, 3*time.Second, 100*time.Millisecond) @@ -1660,8 +1660,8 @@ func (s *TracerSuite) TestSendfileRegression() { t.Logf("looking for connections %+v <-> %+v", c.LocalAddr(), c.RemoteAddr()) var outConn, inConn *network.ConnectionStats - assert.Eventually(t, func() bool { - conns := getConnections(t, tr) + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + conns := getConnections(ct, tr) t.Log(conns) if outConn == nil { outConn = network.FirstConnection(conns, network.ByType(connType), network.ByFamily(family), network.ByTuple(c.LocalAddr(), c.RemoteAddr())) @@ -1669,7 +1669,8 @@ func (s *TracerSuite) TestSendfileRegression() { if inConn == nil { inConn = network.FirstConnection(conns, network.ByType(connType), network.ByFamily(family), network.ByTuple(c.RemoteAddr(), c.LocalAddr())) } - return outConn != nil && inConn != nil + require.NotNil(ct, outConn) + require.NotNil(ct, inConn) }, 3*time.Second, 100*time.Millisecond, "couldn't find connections used by sendfile(2)") if assert.NotNil(t, outConn, "couldn't find outgoing connection used by sendfile(2)") { @@ -1979,16 +1980,16 @@ func (s *TracerSuite) TestPreexistingConnectionDirection() { t := s.T() // Start the client and server before we enable the system probe to test that the tracer picks // up the pre-existing connection - server := tracertestutil.NewTCPServer(func(c net.Conn) { r := bufio.NewReader(c) for { if _, err := r.ReadBytes(byte('\n')); err != nil { assert.ErrorIs(t, err, io.EOF, "exited server loop with error that is not EOF") - return + break } _, _ = c.Write(genPayload(serverMessageSize)) } + c.Close() }) t.Cleanup(server.Shutdown) require.NoError(t, server.Run()) @@ -2023,37 +2024,55 @@ func (s *TracerSuite) TestPreexistingConnectionDirection() { } require.NotNil(collect, outgoing) require.NotNil(collect, incoming) - }, 3*time.Second, 100*time.Millisecond, "could not find connection incoming and outgoing connections") + if !assert.True(collect, incoming != nil && outgoing != nil) { + return + } - m := outgoing.Monotonic - assert.Equal(t, clientMessageSize, int(m.SentBytes)) - assert.Equal(t, serverMessageSize, int(m.RecvBytes)) - if !tr.config.EnableEbpfless { - assert.Equal(t, os.Getpid(), int(outgoing.Pid)) - } - assert.Equal(t, addrPort(server.Address()), int(outgoing.DPort)) - assert.Equal(t, c.LocalAddr().(*net.TCPAddr).Port, int(outgoing.SPort)) - assert.Equal(t, network.OUTGOING, outgoing.Direction) + m := outgoing.Monotonic + assert.Equal(collect, clientMessageSize, int(m.SentBytes)) + // ebpfless RecvBytes is based off acknowledgements, so it can miss the first + // packet in a pre-existing connection + if !tr.config.EnableEbpfless { + assert.Equal(collect, serverMessageSize, int(m.RecvBytes)) + } + if !tr.config.EnableEbpfless { + assert.Equal(collect, os.Getpid(), int(outgoing.Pid)) + } + assert.Equal(collect, addrPort(server.Address()), int(outgoing.DPort)) + assert.Equal(collect, c.LocalAddr().(*net.TCPAddr).Port, int(outgoing.SPort)) + assert.Equal(collect, network.OUTGOING, outgoing.Direction) + + m = incoming.Monotonic + // ebpfless RecvBytes is based off acknowledgements, so it can miss the first + // packet in a pre-existing connection + if !tr.config.EnableEbpfless { + assert.Equal(collect, clientMessageSize, int(m.RecvBytes)) + } + assert.Equal(collect, serverMessageSize, int(m.SentBytes)) + if !tr.config.EnableEbpfless { + assert.Equal(collect, os.Getpid(), int(incoming.Pid)) + } + assert.Equal(collect, addrPort(server.Address()), int(incoming.SPort)) + assert.Equal(collect, c.LocalAddr().(*net.TCPAddr).Port, int(incoming.DPort)) + assert.Equal(collect, network.INCOMING, incoming.Direction) + }, 3*time.Second, 100*time.Millisecond, "could not find connection incoming and outgoing connections") - m = incoming.Monotonic - assert.Equal(t, clientMessageSize, int(m.RecvBytes)) - assert.Equal(t, serverMessageSize, int(m.SentBytes)) - if !tr.config.EnableEbpfless { - assert.Equal(t, os.Getpid(), int(incoming.Pid)) - } - assert.Equal(t, addrPort(server.Address()), int(incoming.SPort)) - assert.Equal(t, c.LocalAddr().(*net.TCPAddr).Port, int(incoming.DPort)) - assert.Equal(t, network.INCOMING, incoming.Direction) } func (s *TracerSuite) TestPreexistingEmptyIncomingConnectionDirection() { t := s.T() + + // The ebpf tracer uses this to ensure it drops pre-existing connections + // that close empty (with no data), because they are difficult to track. + // However, in ebpfless they are easy to track, so disable this test. + // For more context, see PR #31100 + skipOnEbpflessNotSupported(t, testConfig()) + t.Run("ringbuf_enabled", func(t *testing.T) { if features.HaveMapType(ebpf.RingBuf) != nil { t.Skip("skipping test as ringbuffers are not supported on this kernel") } c := testConfig() - skipOnEbpflessNotSupported(t, c) c.NPMRingbuffersEnabled = true testPreexistingEmptyIncomingConnectionDirection(t, c) }) @@ -2415,7 +2434,7 @@ LOOP: require.NoError(t, c.Close(), "error closing client connection") require.EventuallyWithT(t, func(collect *assert.CollectT) { - conn, found := findConnection(c.LocalAddr(), srv.Addr(), getConnections(t, tr)) + conn, found := findConnection(c.LocalAddr(), srv.Addr(), getConnections(collect, tr)) require.True(collect, found, "could not find connection") require.True(collect, conn.IsClosed, "connection should be closed") // after closing the client connection, the duration should be diff --git a/pkg/network/tracer/tracer_test.go b/pkg/network/tracer/tracer_test.go index 710e5eb142253..d50f4f299690c 100644 --- a/pkg/network/tracer/tracer_test.go +++ b/pkg/network/tracer/tracer_test.go @@ -1130,6 +1130,7 @@ func (s *TracerSuite) TestTCPEstablishedPreExistingConn() { c, err := net.DialTimeout("tcp", server.Address(), 50*time.Millisecond) require.NoError(t, err) laddr, raddr := c.LocalAddr(), c.RemoteAddr() + t.Logf("laddr=%s raddr=%s", laddr, raddr) // Ensure closed connections are flushed as soon as possible cfg := testConfig() diff --git a/pkg/network/usm/debugger/README.md b/pkg/network/usm/debugger/README.md new file mode 100644 index 0000000000000..27d71055a8e31 --- /dev/null +++ b/pkg/network/usm/debugger/README.md @@ -0,0 +1,31 @@ +# USM debugger + +A minimal, self-contained build of USM for faster iterations of debugging eBPF +code on remote machines. + +Prepare the build for the target architectures you need (the `foo` KMT stack +does not need to exist). This is needed in order to be able to build the eBPF +object files for a different architecture than your host machine. + +``` +inv -e kmt.prepare system-probe --compile-only --stack=foo --arch=x86_64 +inv -e kmt.prepare system-probe --compile-only --stack=foo --arch=arm64 +``` + +Build the binary with one of the following commands for the architecture of +your target machine: + +``` +inv -e system-probe.build-usm-debugger --arch=x86_64 +inv -e system-probe.build-usm-debugger --arch=arm64 +``` + +Copy the `bin/usm-debugger` to the `system-probe` container in the +`datadog-agent` pod on your target machine. Open a shell in that container and +execute the binary. + +The eBPF programs are always built with debug logs enabled so you can view them +with `cat /sys/kernel/tracing/trace_pipe`. + +If you need to change the system-probe config, edit `cmd/usm_debugger.go` and +rebuild. diff --git a/pkg/network/usm/debugger/cmd/.gitignore b/pkg/network/usm/debugger/cmd/.gitignore new file mode 100644 index 0000000000000..5761abcfdf0c2 --- /dev/null +++ b/pkg/network/usm/debugger/cmd/.gitignore @@ -0,0 +1 @@ +*.o diff --git a/pkg/network/usm/debugger/cmd/ebpf_bytecode.go b/pkg/network/usm/debugger/cmd/ebpf_bytecode.go new file mode 100644 index 0000000000000..e924386d9278d --- /dev/null +++ b/pkg/network/usm/debugger/cmd/ebpf_bytecode.go @@ -0,0 +1,62 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build usm_debugger + +package main + +import ( + _ "embed" + "os" + "path" + + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// The code below is essentially responsible for embedding the CO-RE artifacts +// during compilation time and writing them to a temporary folder during +// runtime, so they can be loaded by the `usm.Monitor` as regular compilation +// assets. + +//go:embed usm-debug.o +var usmProgram []byte + +//go:embed shared-libraries-debug.o +var sharedLibrariesProgram []byte + +func setupBytecode() func() { + type program struct { + filePath string + bytecode []byte + } + + var ( + bytecodeDir = os.TempDir() + coreDir = path.Join(bytecodeDir, "co-re") + ) + + os.Setenv("DD_SYSTEM_PROBE_BPF_DIR", bytecodeDir) + err := os.MkdirAll(coreDir, os.ModePerm) + checkError(err) + + programs := []program{ + {path.Join(coreDir, "usm-debug.o"), usmProgram}, + {path.Join(coreDir, "shared-libraries-debug.o"), sharedLibrariesProgram}, + } + + for _, p := range programs { + f, err := os.Create(p.filePath) + checkError(err) + _, err = f.Write(p.bytecode) + checkError(err) + log.Debugf("writing ebpf bytecode to %s", p.filePath) + } + + return func() { + for _, p := range programs { + os.Remove(p.filePath) + } + } +} diff --git a/pkg/network/usm/debugger/cmd/usm_debugger.go b/pkg/network/usm/debugger/cmd/usm_debugger.go new file mode 100644 index 0000000000000..70fb985c169be --- /dev/null +++ b/pkg/network/usm/debugger/cmd/usm_debugger.go @@ -0,0 +1,89 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build usm_debugger + +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/DataDog/datadog-agent/cmd/system-probe/config" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + networkconfig "github.com/DataDog/datadog-agent/pkg/network/config" + "github.com/DataDog/datadog-agent/pkg/network/usm" + pkglogsetup "github.com/DataDog/datadog-agent/pkg/util/log/setup" +) + +func main() { + err := pkglogsetup.SetupLogger( + "usm-debugger", + "debug", + "", + pkglogsetup.GetSyslogURI(pkgconfigsetup.Datadog()), + false, + true, + false, + pkgconfigsetup.Datadog(), + ) + checkError(err) + + cleanupFn := setupBytecode() + defer cleanupFn() + + monitor, err := usm.NewMonitor(getConfiguration(), nil) + checkError(err) + + err = monitor.Start() + checkError(err) + + go func() { + t := time.NewTicker(10 * time.Second) + for range t.C { + _ = monitor.GetProtocolStats() + } + }() + + defer monitor.Stop() + done := make(chan os.Signal, 1) + signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) + <-done +} + +func checkError(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(-1) + } +} + +func getConfiguration() *networkconfig.Config { + // done for the purposes of initializing the configuration values + _, err := config.New("", "") + checkError(err) + + c := networkconfig.New() + + // run debug version of the eBPF program + c.BPFDebug = true + c.EnableUSMEventStream = false + + // don't buffer data in userspace + // this is to ensure that we won't inadvertently trigger an OOM kill + // by enabling the debugger inside a system-probe container. + c.MaxHTTPStatsBuffered = 0 + c.MaxKafkaStatsBuffered = 0 + + // make sure we use the CO-RE compilation artifact embedded + // in this build (see `ebpf_bytecode.go`) + c.EnableCORE = true + c.EnableRuntimeCompiler = false + + return c +} diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index 64e452ea7fe45..4a7800de41d4b 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -237,7 +237,8 @@ var gnuTLSProbes = []manager.ProbesSelector{ } const ( - sslSockByCtxMap = "ssl_sock_by_ctx" + sslSockByCtxMap = "ssl_sock_by_ctx" + sslCtxByPIDTGIDMap = "ssl_ctx_by_pid_tgid" ) var ( @@ -271,7 +272,7 @@ var opensslSpec = &protocols.ProtocolSpec{ Name: "fd_by_ssl_bio", }, { - Name: "ssl_ctx_by_pid_tgid", + Name: sslCtxByPIDTGIDMap, }, }, Probes: []*manager.Probe{ @@ -491,6 +492,10 @@ func (o *sslProgram) ConfigureOptions(_ *manager.Manager, options *manager.Optio MaxEntries: o.cfg.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries, } + options.MapSpecEditors[sslCtxByPIDTGIDMap] = manager.MapSpecEditor{ + MaxEntries: o.cfg.MaxTrackedConnections, + EditorFlag: manager.EditMaxEntries, + } } // PreStart is called before the start of the provided eBPF manager. @@ -552,7 +557,7 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map) spew.Fdump(w, key, value) } - case "ssl_ctx_by_pid_tgid": // maps/ssl_ctx_by_pid_tgid (BPF_MAP_TYPE_HASH), key C.__u64, value uintptr // C.void * + case sslCtxByPIDTGIDMap: // maps/ssl_ctx_by_pid_tgid (BPF_MAP_TYPE_HASH), key C.__u64, value uintptr // C.void * io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'uintptr // C.void *'\n") iter := currentMap.Iterate() var key uint64 diff --git a/pkg/network/usm/utils/debugger.go b/pkg/network/usm/utils/debugger.go index 4da685bf2fbc9..8094b2e02c6c3 100644 --- a/pkg/network/usm/utils/debugger.go +++ b/pkg/network/usm/utils/debugger.go @@ -31,6 +31,7 @@ type Attacher interface { type TracedProgram struct { ProgramType string FilePath string + PathID PathIdentifier PIDs []uint32 } @@ -131,6 +132,7 @@ func (d *tlsDebugger) GetTracedPrograms(moduleName string) []TracedProgram { program.ProgramType = programType program.FilePath = registration.sampleFilePath + program.PathID = pathID } registry.m.Unlock() diff --git a/pkg/networkdevice/metadata/payload.go b/pkg/networkdevice/metadata/payload.go index 54b2f45eb0996..5f67824728cbe 100644 --- a/pkg/networkdevice/metadata/payload.go +++ b/pkg/networkdevice/metadata/payload.go @@ -131,10 +131,11 @@ type TopologyLinkSide struct { // TopologyLinkMetadata contains topology interface to interface links metadata type TopologyLinkMetadata struct { - ID string `json:"id"` - SourceType string `json:"source_type"` - Local *TopologyLinkSide `json:"local"` - Remote *TopologyLinkSide `json:"remote"` + ID string `json:"id"` + SourceType string `json:"source_type"` + Integration string `json:"integration,omitempty"` + Local *TopologyLinkSide `json:"local"` + Remote *TopologyLinkSide `json:"remote"` } // NetflowExporter contains netflow exporters info diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index b34df591521b5..df547c8beb777 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -20,7 +20,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index 063bd88a07005..87558fb1a5f79 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -62,8 +62,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/orchestrator/model/go.mod b/pkg/orchestrator/model/go.mod index 96f90197c3fe2..f941c7960e5ae 100644 --- a/pkg/orchestrator/model/go.mod +++ b/pkg/orchestrator/model/go.mod @@ -17,7 +17,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/orchestrator/model/go.sum b/pkg/orchestrator/model/go.sum index 3c93bebd8c6dd..01c4e89a188d8 100644 --- a/pkg/orchestrator/model/go.sum +++ b/pkg/orchestrator/model/go.sum @@ -19,7 +19,5 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/proto/datadog/api/v1/api.proto b/pkg/proto/datadog/api/v1/api.proto index 1ffbfa02aa562..3e53e3aa7c773 100644 --- a/pkg/proto/datadog/api/v1/api.proto +++ b/pkg/proto/datadog/api/v1/api.proto @@ -51,6 +51,22 @@ service AgentSecure { }; }; + // Generates a container ID from Origin Info. + // can be called through the HTTP gateway, and entity will be returned as JSON: + // $ curl -H "authorization: Bearer $(cat /etc/datadog-agent/auth_token)" \ + // -XPOST -k -H "Content-Type: application/json" \ + // --data '{"externalDataInit": false,"externalDataPodUID": "54383382-cea3-49e3-9dda-325436ddd5b8","externalDataContainerName": "dd-trace-py"}' \ + // https://localhost:5001/v1/grpc/tagger/generate_container_id_from_origin_info + // { + // "containerID":"c9fd60251b5237467462dad48999815eb0025f367c6e1abe91e0bd787d5915fc" + // } + rpc TaggerGenerateContainerIDFromOriginInfo(datadog.model.v1.GenerateContainerIDFromOriginInfoRequest) returns (datadog.model.v1.GenerateContainerIDFromOriginInfoResponse) { + option (google.api.http) = { + post: "/v1/grpc/tagger/generate_container_id_from_origin_info" + body: "*" + }; + }; + // fetches an entity from the Tagger with the desired cardinality tags. // can be called through the HTTP gateway, and entity will be returned as JSON: // $ curl -H "authorization: Bearer $(cat /etc/datadog-agent/auth_token)" \ diff --git a/pkg/proto/datadog/model/v1/model.proto b/pkg/proto/datadog/model/v1/model.proto index 506f90fa95702..2d4f96d208483 100644 --- a/pkg/proto/datadog/model/v1/model.proto +++ b/pkg/proto/datadog/model/v1/model.proto @@ -73,6 +73,30 @@ message Entity { repeated string standardTags = 6; } +message GenerateContainerIDFromOriginInfoRequest { + // Nested message for the local data + message LocalData { + optional uint32 processID = 1; // Process ID of the container process on the host. + optional string containerID = 2; // Container ID send from the client. + optional uint64 inode = 3; // Cgroup inode of the container. + optional string podUID = 4; // Pod UID send from the client. + } + + // Nested message for the external data + message ExternalData { + optional bool init = 1; // Init is true if the container is an init container. + optional string containerName = 2; // Container name in the Kubernetes Pod spec. + optional string podUID = 3; // Pod UID in the Kubernetes Pod spec. + } + + optional LocalData localData = 1; // Local data for the container, generated by the client. + optional ExternalData externalData = 2; // External data for the container, generated by the Admission Controller. +} + +message GenerateContainerIDFromOriginInfoResponse { + string containerID = 1; +} + message FetchEntityRequest { EntityId id = 1; TagCardinality cardinality = 2; diff --git a/pkg/proto/go.mod b/pkg/proto/go.mod index 283fed1972634..890f1f5360dcf 100644 --- a/pkg/proto/go.mod +++ b/pkg/proto/go.mod @@ -25,7 +25,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/pkg/proto/go.sum b/pkg/proto/go.sum index 899e2f639d24b..bbbf81762e92d 100644 --- a/pkg/proto/go.sum +++ b/pkg/proto/go.sum @@ -91,8 +91,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/proto/pbgo/core/api.pb.go b/pkg/proto/pbgo/core/api.pb.go index 57de2030c617e..f99b9d0f48fbf 100644 --- a/pkg/proto/pbgo/core/api.pb.go +++ b/pkg/proto/pbgo/core/api.pb.go @@ -53,7 +53,7 @@ var file_datadog_api_v1_api_proto_rawDesc = []byte{ 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x15, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0f, 0x12, 0x0d, 0x2f, 0x76, 0x31, - 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68, 0x6f, 0x73, 0x74, 0x32, 0x80, 0x0d, 0x0a, 0x0b, 0x41, + 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68, 0x6f, 0x73, 0x74, 0x32, 0xe8, 0x0e, 0x0a, 0x0b, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x75, 0x72, 0x65, 0x12, 0x8f, 0x01, 0x0a, 0x14, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x23, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, @@ -63,176 +63,194 @@ var file_datadog_api_v1_api_proto_rawDesc = []byte{ 0x61, 0x6d, 0x54, 0x61, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2a, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x24, 0x3a, 0x01, 0x2a, 0x22, 0x1f, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x30, 0x01, 0x12, 0x89, 0x01, 0x0a, - 0x11, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x12, 0x24, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, - 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x65, 0x74, 0x63, - 0x68, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x27, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x21, 0x3a, 0x01, 0x2a, 0x22, 0x1c, 0x2f, 0x76, 0x31, 0x2f, - 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x66, 0x65, 0x74, 0x63, - 0x68, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x9b, 0x01, 0x0a, 0x17, 0x44, 0x6f, 0x67, - 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x54, 0x72, 0x69, - 0x67, 0x67, 0x65, 0x72, 0x12, 0x27, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, - 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x54, - 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, - 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2d, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x27, 0x3a, - 0x01, 0x2a, 0x22, 0x22, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x6f, 0x67, - 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x2f, 0x63, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x2f, 0x74, - 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x8c, 0x01, 0x0a, 0x17, 0x44, 0x6f, 0x67, 0x73, 0x74, - 0x61, 0x74, 0x73, 0x64, 0x53, 0x65, 0x74, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x1a, 0x25, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x25, - 0x3a, 0x01, 0x2a, 0x22, 0x20, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x6f, - 0x67, 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x2f, 0x63, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x2f, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x8f, 0x01, 0x0a, 0x10, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x27, 0x2e, 0x64, 0x61, 0x74, - 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, - 0xd3, 0xe4, 0x93, 0x02, 0x22, 0x3a, 0x01, 0x2a, 0x22, 0x1d, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, - 0x70, 0x63, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x78, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x1a, 0x26, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x94, 0x01, 0x0a, 0x12, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x48, 0x41, 0x12, 0x27, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, - 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x25, 0x3a, 0x01, 0x2a, 0x22, 0x20, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, - 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x68, 0x61, 0x12, 0x7d, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, 0x41, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x26, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x82, 0xd3, - 0xe4, 0x93, 0x02, 0x23, 0x3a, 0x01, 0x2a, 0x22, 0x1e, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, - 0x63, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x5f, 0x68, 0x61, 0x12, 0xb3, 0x01, 0x0a, 0x1a, 0x57, 0x6f, 0x72, 0x6b, - 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x2f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, - 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x57, 0x6f, - 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, - 0x67, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x57, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x30, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x2a, 0x3a, 0x01, 0x2a, 0x22, 0x25, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x77, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, 0x2f, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x30, 0x01, 0x12, 0xaf, 0x01, - 0x0a, 0x13, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, - 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x67, - 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x35, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2f, - 0x3a, 0x01, 0x2a, 0x22, 0x2a, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x12, - 0x9b, 0x01, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, - 0x79, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x32, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, - 0x61, 0x75, 0x74, 0x6f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x41, 0x75, - 0x74, 0x6f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x30, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x2a, 0x3a, 0x01, 0x2a, 0x22, 0x25, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x61, - 0x75, 0x74, 0x6f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2f, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x30, 0x01, 0x32, 0xe6, 0x01, - 0x0a, 0x0b, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x6f, 0x0a, - 0x10, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x12, 0x2c, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, - 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, - 0x29, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x72, 0x65, 0x46, 0x69, - 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x64, 0x61, 0x74, - 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, - 0x2e, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x15, 0x5a, 0x13, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x67, 0x6f, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6d, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x30, 0x01, 0x12, 0xe5, 0x01, 0x0a, + 0x27, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x72, + 0x69, 0x67, 0x69, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3a, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, + 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x46, + 0x72, 0x6f, 0x6d, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x46, 0x72, 0x6f, 0x6d, 0x4f, + 0x72, 0x69, 0x67, 0x69, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x41, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3b, 0x3a, 0x01, 0x2a, 0x22, 0x36, 0x2f, 0x76, + 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x5f, + 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x89, 0x01, 0x0a, 0x11, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x46, + 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x24, 0x2e, 0x64, 0x61, 0x74, + 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x65, + 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x25, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x27, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x21, 0x3a, + 0x01, 0x2a, 0x22, 0x1c, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x61, 0x67, + 0x67, 0x65, 0x72, 0x2f, 0x66, 0x65, 0x74, 0x63, 0x68, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x12, 0x9b, 0x01, 0x0a, 0x17, 0x44, 0x6f, 0x67, 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x43, 0x61, + 0x70, 0x74, 0x75, 0x72, 0x65, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x27, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, + 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x2d, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x27, 0x3a, 0x01, 0x2a, 0x22, 0x22, 0x2f, 0x76, 0x31, 0x2f, + 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x6f, 0x67, 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x2f, 0x63, + 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x2f, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x8c, + 0x01, 0x0a, 0x17, 0x44, 0x6f, 0x67, 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x53, 0x65, 0x74, 0x54, + 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, + 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, + 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x25, 0x2e, 0x64, 0x61, 0x74, 0x61, + 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, + 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x2b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x25, 0x3a, 0x01, 0x2a, 0x22, 0x20, 0x2f, 0x76, 0x31, + 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x6f, 0x67, 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x2f, + 0x63, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x8f, 0x01, + 0x0a, 0x10, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x73, 0x12, 0x27, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x22, 0x3a, 0x01, 0x2a, + 0x22, 0x1d, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, + 0x78, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x26, 0x2e, 0x64, 0x61, 0x74, 0x61, + 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, 0x76, + 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x94, 0x01, 0x0a, 0x12, 0x43, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x48, 0x41, + 0x12, 0x27, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x61, 0x74, 0x61, + 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x25, 0x3a, 0x01, 0x2a, 0x22, 0x20, + 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x68, 0x61, + 0x12, 0x7d, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x48, 0x41, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x26, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x23, 0x3a, 0x01, 0x2a, 0x22, + 0x1e, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x68, 0x61, 0x12, + 0xb3, 0x01, 0x0a, 0x1a, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x2f, + 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, 0x65, + 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x30, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, + 0x65, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x30, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2a, 0x3a, 0x01, 0x2a, 0x22, 0x25, 0x2f, 0x76, + 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x6d, + 0x65, 0x74, 0x61, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x30, 0x01, 0x12, 0xaf, 0x01, 0x0a, 0x13, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x65, 0x72, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x2e, + 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, + 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, + 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x35, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2f, 0x3a, 0x01, 0x2a, 0x22, 0x2a, 0x2f, 0x76, 0x31, + 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, + 0x74, 0x2f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x9b, 0x01, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, + 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x32, 0x2e, + 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x64, 0x69, 0x73, 0x63, + 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x79, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x30, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2a, 0x3a, 0x01, 0x2a, 0x22, 0x25, 0x2f, 0x76, + 0x31, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x64, 0x69, 0x73, 0x63, 0x6f, + 0x76, 0x65, 0x72, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x73, 0x30, 0x01, 0x32, 0xe6, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x6f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x2c, 0x2e, 0x64, 0x61, 0x74, 0x61, + 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, + 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, + 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, + 0x67, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, + 0x74, 0x46, 0x6c, 0x61, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x72, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x15, + 0x5a, 0x13, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x67, 0x6f, + 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var file_datadog_api_v1_api_proto_goTypes = []any{ - (*HostnameRequest)(nil), // 0: datadog.model.v1.HostnameRequest - (*StreamTagsRequest)(nil), // 1: datadog.model.v1.StreamTagsRequest - (*FetchEntityRequest)(nil), // 2: datadog.model.v1.FetchEntityRequest - (*CaptureTriggerRequest)(nil), // 3: datadog.model.v1.CaptureTriggerRequest - (*TaggerState)(nil), // 4: datadog.model.v1.TaggerState - (*ClientGetConfigsRequest)(nil), // 5: datadog.config.ClientGetConfigsRequest - (*empty.Empty)(nil), // 6: google.protobuf.Empty - (*WorkloadmetaStreamRequest)(nil), // 7: datadog.workloadmeta.WorkloadmetaStreamRequest - (*RegisterRemoteAgentRequest)(nil), // 8: datadog.remoteagent.RegisterRemoteAgentRequest - (*GetStatusDetailsRequest)(nil), // 9: datadog.remoteagent.GetStatusDetailsRequest - (*GetFlareFilesRequest)(nil), // 10: datadog.remoteagent.GetFlareFilesRequest - (*HostnameReply)(nil), // 11: datadog.model.v1.HostnameReply - (*StreamTagsResponse)(nil), // 12: datadog.model.v1.StreamTagsResponse - (*FetchEntityResponse)(nil), // 13: datadog.model.v1.FetchEntityResponse - (*CaptureTriggerResponse)(nil), // 14: datadog.model.v1.CaptureTriggerResponse - (*TaggerStateResponse)(nil), // 15: datadog.model.v1.TaggerStateResponse - (*ClientGetConfigsResponse)(nil), // 16: datadog.config.ClientGetConfigsResponse - (*GetStateConfigResponse)(nil), // 17: datadog.config.GetStateConfigResponse - (*WorkloadmetaStreamResponse)(nil), // 18: datadog.workloadmeta.WorkloadmetaStreamResponse - (*RegisterRemoteAgentResponse)(nil), // 19: datadog.remoteagent.RegisterRemoteAgentResponse - (*AutodiscoveryStreamResponse)(nil), // 20: datadog.autodiscovery.AutodiscoveryStreamResponse - (*GetStatusDetailsResponse)(nil), // 21: datadog.remoteagent.GetStatusDetailsResponse - (*GetFlareFilesResponse)(nil), // 22: datadog.remoteagent.GetFlareFilesResponse + (*HostnameRequest)(nil), // 0: datadog.model.v1.HostnameRequest + (*StreamTagsRequest)(nil), // 1: datadog.model.v1.StreamTagsRequest + (*GenerateContainerIDFromOriginInfoRequest)(nil), // 2: datadog.model.v1.GenerateContainerIDFromOriginInfoRequest + (*FetchEntityRequest)(nil), // 3: datadog.model.v1.FetchEntityRequest + (*CaptureTriggerRequest)(nil), // 4: datadog.model.v1.CaptureTriggerRequest + (*TaggerState)(nil), // 5: datadog.model.v1.TaggerState + (*ClientGetConfigsRequest)(nil), // 6: datadog.config.ClientGetConfigsRequest + (*empty.Empty)(nil), // 7: google.protobuf.Empty + (*WorkloadmetaStreamRequest)(nil), // 8: datadog.workloadmeta.WorkloadmetaStreamRequest + (*RegisterRemoteAgentRequest)(nil), // 9: datadog.remoteagent.RegisterRemoteAgentRequest + (*GetStatusDetailsRequest)(nil), // 10: datadog.remoteagent.GetStatusDetailsRequest + (*GetFlareFilesRequest)(nil), // 11: datadog.remoteagent.GetFlareFilesRequest + (*HostnameReply)(nil), // 12: datadog.model.v1.HostnameReply + (*StreamTagsResponse)(nil), // 13: datadog.model.v1.StreamTagsResponse + (*GenerateContainerIDFromOriginInfoResponse)(nil), // 14: datadog.model.v1.GenerateContainerIDFromOriginInfoResponse + (*FetchEntityResponse)(nil), // 15: datadog.model.v1.FetchEntityResponse + (*CaptureTriggerResponse)(nil), // 16: datadog.model.v1.CaptureTriggerResponse + (*TaggerStateResponse)(nil), // 17: datadog.model.v1.TaggerStateResponse + (*ClientGetConfigsResponse)(nil), // 18: datadog.config.ClientGetConfigsResponse + (*GetStateConfigResponse)(nil), // 19: datadog.config.GetStateConfigResponse + (*WorkloadmetaStreamResponse)(nil), // 20: datadog.workloadmeta.WorkloadmetaStreamResponse + (*RegisterRemoteAgentResponse)(nil), // 21: datadog.remoteagent.RegisterRemoteAgentResponse + (*AutodiscoveryStreamResponse)(nil), // 22: datadog.autodiscovery.AutodiscoveryStreamResponse + (*GetStatusDetailsResponse)(nil), // 23: datadog.remoteagent.GetStatusDetailsResponse + (*GetFlareFilesResponse)(nil), // 24: datadog.remoteagent.GetFlareFilesResponse } var file_datadog_api_v1_api_proto_depIdxs = []int32{ 0, // 0: datadog.api.v1.Agent.GetHostname:input_type -> datadog.model.v1.HostnameRequest 1, // 1: datadog.api.v1.AgentSecure.TaggerStreamEntities:input_type -> datadog.model.v1.StreamTagsRequest - 2, // 2: datadog.api.v1.AgentSecure.TaggerFetchEntity:input_type -> datadog.model.v1.FetchEntityRequest - 3, // 3: datadog.api.v1.AgentSecure.DogstatsdCaptureTrigger:input_type -> datadog.model.v1.CaptureTriggerRequest - 4, // 4: datadog.api.v1.AgentSecure.DogstatsdSetTaggerState:input_type -> datadog.model.v1.TaggerState - 5, // 5: datadog.api.v1.AgentSecure.ClientGetConfigs:input_type -> datadog.config.ClientGetConfigsRequest - 6, // 6: datadog.api.v1.AgentSecure.GetConfigState:input_type -> google.protobuf.Empty - 5, // 7: datadog.api.v1.AgentSecure.ClientGetConfigsHA:input_type -> datadog.config.ClientGetConfigsRequest - 6, // 8: datadog.api.v1.AgentSecure.GetConfigStateHA:input_type -> google.protobuf.Empty - 7, // 9: datadog.api.v1.AgentSecure.WorkloadmetaStreamEntities:input_type -> datadog.workloadmeta.WorkloadmetaStreamRequest - 8, // 10: datadog.api.v1.AgentSecure.RegisterRemoteAgent:input_type -> datadog.remoteagent.RegisterRemoteAgentRequest - 6, // 11: datadog.api.v1.AgentSecure.AutodiscoveryStreamConfig:input_type -> google.protobuf.Empty - 9, // 12: datadog.api.v1.RemoteAgent.GetStatusDetails:input_type -> datadog.remoteagent.GetStatusDetailsRequest - 10, // 13: datadog.api.v1.RemoteAgent.GetFlareFiles:input_type -> datadog.remoteagent.GetFlareFilesRequest - 11, // 14: datadog.api.v1.Agent.GetHostname:output_type -> datadog.model.v1.HostnameReply - 12, // 15: datadog.api.v1.AgentSecure.TaggerStreamEntities:output_type -> datadog.model.v1.StreamTagsResponse - 13, // 16: datadog.api.v1.AgentSecure.TaggerFetchEntity:output_type -> datadog.model.v1.FetchEntityResponse - 14, // 17: datadog.api.v1.AgentSecure.DogstatsdCaptureTrigger:output_type -> datadog.model.v1.CaptureTriggerResponse - 15, // 18: datadog.api.v1.AgentSecure.DogstatsdSetTaggerState:output_type -> datadog.model.v1.TaggerStateResponse - 16, // 19: datadog.api.v1.AgentSecure.ClientGetConfigs:output_type -> datadog.config.ClientGetConfigsResponse - 17, // 20: datadog.api.v1.AgentSecure.GetConfigState:output_type -> datadog.config.GetStateConfigResponse - 16, // 21: datadog.api.v1.AgentSecure.ClientGetConfigsHA:output_type -> datadog.config.ClientGetConfigsResponse - 17, // 22: datadog.api.v1.AgentSecure.GetConfigStateHA:output_type -> datadog.config.GetStateConfigResponse - 18, // 23: datadog.api.v1.AgentSecure.WorkloadmetaStreamEntities:output_type -> datadog.workloadmeta.WorkloadmetaStreamResponse - 19, // 24: datadog.api.v1.AgentSecure.RegisterRemoteAgent:output_type -> datadog.remoteagent.RegisterRemoteAgentResponse - 20, // 25: datadog.api.v1.AgentSecure.AutodiscoveryStreamConfig:output_type -> datadog.autodiscovery.AutodiscoveryStreamResponse - 21, // 26: datadog.api.v1.RemoteAgent.GetStatusDetails:output_type -> datadog.remoteagent.GetStatusDetailsResponse - 22, // 27: datadog.api.v1.RemoteAgent.GetFlareFiles:output_type -> datadog.remoteagent.GetFlareFilesResponse - 14, // [14:28] is the sub-list for method output_type - 0, // [0:14] is the sub-list for method input_type + 2, // 2: datadog.api.v1.AgentSecure.TaggerGenerateContainerIDFromOriginInfo:input_type -> datadog.model.v1.GenerateContainerIDFromOriginInfoRequest + 3, // 3: datadog.api.v1.AgentSecure.TaggerFetchEntity:input_type -> datadog.model.v1.FetchEntityRequest + 4, // 4: datadog.api.v1.AgentSecure.DogstatsdCaptureTrigger:input_type -> datadog.model.v1.CaptureTriggerRequest + 5, // 5: datadog.api.v1.AgentSecure.DogstatsdSetTaggerState:input_type -> datadog.model.v1.TaggerState + 6, // 6: datadog.api.v1.AgentSecure.ClientGetConfigs:input_type -> datadog.config.ClientGetConfigsRequest + 7, // 7: datadog.api.v1.AgentSecure.GetConfigState:input_type -> google.protobuf.Empty + 6, // 8: datadog.api.v1.AgentSecure.ClientGetConfigsHA:input_type -> datadog.config.ClientGetConfigsRequest + 7, // 9: datadog.api.v1.AgentSecure.GetConfigStateHA:input_type -> google.protobuf.Empty + 8, // 10: datadog.api.v1.AgentSecure.WorkloadmetaStreamEntities:input_type -> datadog.workloadmeta.WorkloadmetaStreamRequest + 9, // 11: datadog.api.v1.AgentSecure.RegisterRemoteAgent:input_type -> datadog.remoteagent.RegisterRemoteAgentRequest + 7, // 12: datadog.api.v1.AgentSecure.AutodiscoveryStreamConfig:input_type -> google.protobuf.Empty + 10, // 13: datadog.api.v1.RemoteAgent.GetStatusDetails:input_type -> datadog.remoteagent.GetStatusDetailsRequest + 11, // 14: datadog.api.v1.RemoteAgent.GetFlareFiles:input_type -> datadog.remoteagent.GetFlareFilesRequest + 12, // 15: datadog.api.v1.Agent.GetHostname:output_type -> datadog.model.v1.HostnameReply + 13, // 16: datadog.api.v1.AgentSecure.TaggerStreamEntities:output_type -> datadog.model.v1.StreamTagsResponse + 14, // 17: datadog.api.v1.AgentSecure.TaggerGenerateContainerIDFromOriginInfo:output_type -> datadog.model.v1.GenerateContainerIDFromOriginInfoResponse + 15, // 18: datadog.api.v1.AgentSecure.TaggerFetchEntity:output_type -> datadog.model.v1.FetchEntityResponse + 16, // 19: datadog.api.v1.AgentSecure.DogstatsdCaptureTrigger:output_type -> datadog.model.v1.CaptureTriggerResponse + 17, // 20: datadog.api.v1.AgentSecure.DogstatsdSetTaggerState:output_type -> datadog.model.v1.TaggerStateResponse + 18, // 21: datadog.api.v1.AgentSecure.ClientGetConfigs:output_type -> datadog.config.ClientGetConfigsResponse + 19, // 22: datadog.api.v1.AgentSecure.GetConfigState:output_type -> datadog.config.GetStateConfigResponse + 18, // 23: datadog.api.v1.AgentSecure.ClientGetConfigsHA:output_type -> datadog.config.ClientGetConfigsResponse + 19, // 24: datadog.api.v1.AgentSecure.GetConfigStateHA:output_type -> datadog.config.GetStateConfigResponse + 20, // 25: datadog.api.v1.AgentSecure.WorkloadmetaStreamEntities:output_type -> datadog.workloadmeta.WorkloadmetaStreamResponse + 21, // 26: datadog.api.v1.AgentSecure.RegisterRemoteAgent:output_type -> datadog.remoteagent.RegisterRemoteAgentResponse + 22, // 27: datadog.api.v1.AgentSecure.AutodiscoveryStreamConfig:output_type -> datadog.autodiscovery.AutodiscoveryStreamResponse + 23, // 28: datadog.api.v1.RemoteAgent.GetStatusDetails:output_type -> datadog.remoteagent.GetStatusDetailsResponse + 24, // 29: datadog.api.v1.RemoteAgent.GetFlareFiles:output_type -> datadog.remoteagent.GetFlareFilesResponse + 15, // [15:30] is the sub-list for method output_type + 0, // [0:15] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -375,6 +393,17 @@ type AgentSecureClient interface { // } // } TaggerStreamEntities(ctx context.Context, in *StreamTagsRequest, opts ...grpc.CallOption) (AgentSecure_TaggerStreamEntitiesClient, error) + // Generates a container ID from Origin Info. + // can be called through the HTTP gateway, and entity will be returned as JSON: + // + // $ curl -H "authorization: Bearer $(cat /etc/datadog-agent/auth_token)" \ + // -XPOST -k -H "Content-Type: application/json" \ + // --data '{"externalDataInit": false,"externalDataPodUID": "54383382-cea3-49e3-9dda-325436ddd5b8","externalDataContainerName": "dd-trace-py"}' \ + // https://localhost:5001/v1/grpc/tagger/generate_container_id_from_origin_info + // { + // "containerID":"c9fd60251b5237467462dad48999815eb0025f367c6e1abe91e0bd787d5915fc" + // } + TaggerGenerateContainerIDFromOriginInfo(ctx context.Context, in *GenerateContainerIDFromOriginInfoRequest, opts ...grpc.CallOption) (*GenerateContainerIDFromOriginInfoResponse, error) // fetches an entity from the Tagger with the desired cardinality tags. // can be called through the HTTP gateway, and entity will be returned as JSON: // @@ -477,6 +506,15 @@ func (x *agentSecureTaggerStreamEntitiesClient) Recv() (*StreamTagsResponse, err return m, nil } +func (c *agentSecureClient) TaggerGenerateContainerIDFromOriginInfo(ctx context.Context, in *GenerateContainerIDFromOriginInfoRequest, opts ...grpc.CallOption) (*GenerateContainerIDFromOriginInfoResponse, error) { + out := new(GenerateContainerIDFromOriginInfoResponse) + err := c.cc.Invoke(ctx, "/datadog.api.v1.AgentSecure/TaggerGenerateContainerIDFromOriginInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *agentSecureClient) TaggerFetchEntity(ctx context.Context, in *FetchEntityRequest, opts ...grpc.CallOption) (*FetchEntityResponse, error) { out := new(FetchEntityResponse) err := c.cc.Invoke(ctx, "/datadog.api.v1.AgentSecure/TaggerFetchEntity", in, out, opts...) @@ -637,6 +675,17 @@ type AgentSecureServer interface { // } // } TaggerStreamEntities(*StreamTagsRequest, AgentSecure_TaggerStreamEntitiesServer) error + // Generates a container ID from Origin Info. + // can be called through the HTTP gateway, and entity will be returned as JSON: + // + // $ curl -H "authorization: Bearer $(cat /etc/datadog-agent/auth_token)" \ + // -XPOST -k -H "Content-Type: application/json" \ + // --data '{"externalDataInit": false,"externalDataPodUID": "54383382-cea3-49e3-9dda-325436ddd5b8","externalDataContainerName": "dd-trace-py"}' \ + // https://localhost:5001/v1/grpc/tagger/generate_container_id_from_origin_info + // { + // "containerID":"c9fd60251b5237467462dad48999815eb0025f367c6e1abe91e0bd787d5915fc" + // } + TaggerGenerateContainerIDFromOriginInfo(context.Context, *GenerateContainerIDFromOriginInfoRequest) (*GenerateContainerIDFromOriginInfoResponse, error) // fetches an entity from the Tagger with the desired cardinality tags. // can be called through the HTTP gateway, and entity will be returned as JSON: // @@ -706,6 +755,9 @@ type UnimplementedAgentSecureServer struct { func (*UnimplementedAgentSecureServer) TaggerStreamEntities(*StreamTagsRequest, AgentSecure_TaggerStreamEntitiesServer) error { return status.Errorf(codes.Unimplemented, "method TaggerStreamEntities not implemented") } +func (*UnimplementedAgentSecureServer) TaggerGenerateContainerIDFromOriginInfo(context.Context, *GenerateContainerIDFromOriginInfoRequest) (*GenerateContainerIDFromOriginInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method TaggerGenerateContainerIDFromOriginInfo not implemented") +} func (*UnimplementedAgentSecureServer) TaggerFetchEntity(context.Context, *FetchEntityRequest) (*FetchEntityResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method TaggerFetchEntity not implemented") } @@ -762,6 +814,24 @@ func (x *agentSecureTaggerStreamEntitiesServer) Send(m *StreamTagsResponse) erro return x.ServerStream.SendMsg(m) } +func _AgentSecure_TaggerGenerateContainerIDFromOriginInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GenerateContainerIDFromOriginInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentSecureServer).TaggerGenerateContainerIDFromOriginInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/datadog.api.v1.AgentSecure/TaggerGenerateContainerIDFromOriginInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentSecureServer).TaggerGenerateContainerIDFromOriginInfo(ctx, req.(*GenerateContainerIDFromOriginInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _AgentSecure_TaggerFetchEntity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(FetchEntityRequest) if err := dec(in); err != nil { @@ -952,6 +1022,10 @@ var _AgentSecure_serviceDesc = grpc.ServiceDesc{ ServiceName: "datadog.api.v1.AgentSecure", HandlerType: (*AgentSecureServer)(nil), Methods: []grpc.MethodDesc{ + { + MethodName: "TaggerGenerateContainerIDFromOriginInfo", + Handler: _AgentSecure_TaggerGenerateContainerIDFromOriginInfo_Handler, + }, { MethodName: "TaggerFetchEntity", Handler: _AgentSecure_TaggerFetchEntity_Handler, diff --git a/pkg/proto/pbgo/core/api.pb.gw.go b/pkg/proto/pbgo/core/api.pb.gw.go index 0fac62a1daa72..7be078c4eb534 100644 --- a/pkg/proto/pbgo/core/api.pb.gw.go +++ b/pkg/proto/pbgo/core/api.pb.gw.go @@ -77,6 +77,40 @@ func request_AgentSecure_TaggerStreamEntities_0(ctx context.Context, marshaler r } +func request_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0(ctx context.Context, marshaler runtime.Marshaler, client AgentSecureClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq GenerateContainerIDFromOriginInfoRequest + var metadata runtime.ServerMetadata + + newReader, berr := utilities.IOReaderFactory(req.Body) + if berr != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) + } + if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.TaggerGenerateContainerIDFromOriginInfo(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +func local_request_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0(ctx context.Context, marshaler runtime.Marshaler, server AgentSecureServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq GenerateContainerIDFromOriginInfoRequest + var metadata runtime.ServerMetadata + + newReader, berr := utilities.IOReaderFactory(req.Body) + if berr != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) + } + if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := server.TaggerGenerateContainerIDFromOriginInfo(ctx, &protoReq) + return msg, metadata, err + +} + func request_AgentSecure_TaggerFetchEntity_0(ctx context.Context, marshaler runtime.Marshaler, client AgentSecureClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var protoReq FetchEntityRequest var metadata runtime.ServerMetadata @@ -444,6 +478,29 @@ func RegisterAgentSecureHandlerServer(ctx context.Context, mux *runtime.ServeMux return }) + mux.Handle("POST", pattern_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateIncomingContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0(rctx, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + mux.Handle("POST", pattern_AgentSecure_TaggerFetchEntity_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -772,6 +829,26 @@ func RegisterAgentSecureHandlerClient(ctx context.Context, mux *runtime.ServeMux }) + mux.Handle("POST", pattern_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + mux.Handle("POST", pattern_AgentSecure_TaggerFetchEntity_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -978,6 +1055,8 @@ func RegisterAgentSecureHandlerClient(ctx context.Context, mux *runtime.ServeMux var ( pattern_AgentSecure_TaggerStreamEntities_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v1", "grpc", "tagger", "stream_entities"}, "", runtime.AssumeColonVerbOpt(true))) + pattern_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v1", "grpc", "tagger", "generate_container_id_from_origin_info"}, "", runtime.AssumeColonVerbOpt(true))) + pattern_AgentSecure_TaggerFetchEntity_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v1", "grpc", "tagger", "fetch_entity"}, "", runtime.AssumeColonVerbOpt(true))) pattern_AgentSecure_DogstatsdCaptureTrigger_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3, 2, 4}, []string{"v1", "grpc", "dogstatsd", "capture", "trigger"}, "", runtime.AssumeColonVerbOpt(true))) @@ -1002,6 +1081,8 @@ var ( var ( forward_AgentSecure_TaggerStreamEntities_0 = runtime.ForwardResponseStream + forward_AgentSecure_TaggerGenerateContainerIDFromOriginInfo_0 = runtime.ForwardResponseMessage + forward_AgentSecure_TaggerFetchEntity_0 = runtime.ForwardResponseMessage forward_AgentSecure_DogstatsdCaptureTrigger_0 = runtime.ForwardResponseMessage diff --git a/pkg/proto/pbgo/core/model.pb.go b/pkg/proto/pbgo/core/model.pb.go index 3f85da4236b7f..9262b4dc43a5a 100644 --- a/pkg/proto/pbgo/core/model.pb.go +++ b/pkg/proto/pbgo/core/model.pb.go @@ -628,6 +628,104 @@ func (x *Entity) GetStandardTags() []string { return nil } +type GenerateContainerIDFromOriginInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + LocalData *GenerateContainerIDFromOriginInfoRequest_LocalData `protobuf:"bytes,1,opt,name=localData,proto3,oneof" json:"localData,omitempty"` // Local data for the container, generated by the client. + ExternalData *GenerateContainerIDFromOriginInfoRequest_ExternalData `protobuf:"bytes,2,opt,name=externalData,proto3,oneof" json:"externalData,omitempty"` // External data for the container, generated by the Admission Controller. +} + +func (x *GenerateContainerIDFromOriginInfoRequest) Reset() { + *x = GenerateContainerIDFromOriginInfoRequest{} + mi := &file_datadog_model_v1_model_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateContainerIDFromOriginInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateContainerIDFromOriginInfoRequest) ProtoMessage() {} + +func (x *GenerateContainerIDFromOriginInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_datadog_model_v1_model_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateContainerIDFromOriginInfoRequest.ProtoReflect.Descriptor instead. +func (*GenerateContainerIDFromOriginInfoRequest) Descriptor() ([]byte, []int) { + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{9} +} + +func (x *GenerateContainerIDFromOriginInfoRequest) GetLocalData() *GenerateContainerIDFromOriginInfoRequest_LocalData { + if x != nil { + return x.LocalData + } + return nil +} + +func (x *GenerateContainerIDFromOriginInfoRequest) GetExternalData() *GenerateContainerIDFromOriginInfoRequest_ExternalData { + if x != nil { + return x.ExternalData + } + return nil +} + +type GenerateContainerIDFromOriginInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContainerID string `protobuf:"bytes,1,opt,name=containerID,proto3" json:"containerID,omitempty"` +} + +func (x *GenerateContainerIDFromOriginInfoResponse) Reset() { + *x = GenerateContainerIDFromOriginInfoResponse{} + mi := &file_datadog_model_v1_model_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateContainerIDFromOriginInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateContainerIDFromOriginInfoResponse) ProtoMessage() {} + +func (x *GenerateContainerIDFromOriginInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_datadog_model_v1_model_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateContainerIDFromOriginInfoResponse.ProtoReflect.Descriptor instead. +func (*GenerateContainerIDFromOriginInfoResponse) Descriptor() ([]byte, []int) { + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{10} +} + +func (x *GenerateContainerIDFromOriginInfoResponse) GetContainerID() string { + if x != nil { + return x.ContainerID + } + return "" +} + type FetchEntityRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -639,7 +737,7 @@ type FetchEntityRequest struct { func (x *FetchEntityRequest) Reset() { *x = FetchEntityRequest{} - mi := &file_datadog_model_v1_model_proto_msgTypes[9] + mi := &file_datadog_model_v1_model_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -651,7 +749,7 @@ func (x *FetchEntityRequest) String() string { func (*FetchEntityRequest) ProtoMessage() {} func (x *FetchEntityRequest) ProtoReflect() protoreflect.Message { - mi := &file_datadog_model_v1_model_proto_msgTypes[9] + mi := &file_datadog_model_v1_model_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -664,7 +762,7 @@ func (x *FetchEntityRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEntityRequest.ProtoReflect.Descriptor instead. func (*FetchEntityRequest) Descriptor() ([]byte, []int) { - return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{9} + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{11} } func (x *FetchEntityRequest) GetId() *EntityId { @@ -693,7 +791,7 @@ type FetchEntityResponse struct { func (x *FetchEntityResponse) Reset() { *x = FetchEntityResponse{} - mi := &file_datadog_model_v1_model_proto_msgTypes[10] + mi := &file_datadog_model_v1_model_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -705,7 +803,7 @@ func (x *FetchEntityResponse) String() string { func (*FetchEntityResponse) ProtoMessage() {} func (x *FetchEntityResponse) ProtoReflect() protoreflect.Message { - mi := &file_datadog_model_v1_model_proto_msgTypes[10] + mi := &file_datadog_model_v1_model_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -718,7 +816,7 @@ func (x *FetchEntityResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEntityResponse.ProtoReflect.Descriptor instead. func (*FetchEntityResponse) Descriptor() ([]byte, []int) { - return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{10} + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{12} } func (x *FetchEntityResponse) GetId() *EntityId { @@ -753,7 +851,7 @@ type EntityId struct { func (x *EntityId) Reset() { *x = EntityId{} - mi := &file_datadog_model_v1_model_proto_msgTypes[11] + mi := &file_datadog_model_v1_model_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -765,7 +863,7 @@ func (x *EntityId) String() string { func (*EntityId) ProtoMessage() {} func (x *EntityId) ProtoReflect() protoreflect.Message { - mi := &file_datadog_model_v1_model_proto_msgTypes[11] + mi := &file_datadog_model_v1_model_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -778,7 +876,7 @@ func (x *EntityId) ProtoReflect() protoreflect.Message { // Deprecated: Use EntityId.ProtoReflect.Descriptor instead. func (*EntityId) Descriptor() ([]byte, []int) { - return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{11} + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{13} } func (x *EntityId) GetPrefix() string { @@ -812,7 +910,7 @@ type UnixDogstatsdMsg struct { func (x *UnixDogstatsdMsg) Reset() { *x = UnixDogstatsdMsg{} - mi := &file_datadog_model_v1_model_proto_msgTypes[12] + mi := &file_datadog_model_v1_model_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -824,7 +922,7 @@ func (x *UnixDogstatsdMsg) String() string { func (*UnixDogstatsdMsg) ProtoMessage() {} func (x *UnixDogstatsdMsg) ProtoReflect() protoreflect.Message { - mi := &file_datadog_model_v1_model_proto_msgTypes[12] + mi := &file_datadog_model_v1_model_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -837,7 +935,7 @@ func (x *UnixDogstatsdMsg) ProtoReflect() protoreflect.Message { // Deprecated: Use UnixDogstatsdMsg.ProtoReflect.Descriptor instead. func (*UnixDogstatsdMsg) Descriptor() ([]byte, []int) { - return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{12} + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{14} } func (x *UnixDogstatsdMsg) GetTimestamp() int64 { @@ -893,7 +991,7 @@ type TaggerState struct { func (x *TaggerState) Reset() { *x = TaggerState{} - mi := &file_datadog_model_v1_model_proto_msgTypes[13] + mi := &file_datadog_model_v1_model_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -905,7 +1003,7 @@ func (x *TaggerState) String() string { func (*TaggerState) ProtoMessage() {} func (x *TaggerState) ProtoReflect() protoreflect.Message { - mi := &file_datadog_model_v1_model_proto_msgTypes[13] + mi := &file_datadog_model_v1_model_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -918,7 +1016,7 @@ func (x *TaggerState) ProtoReflect() protoreflect.Message { // Deprecated: Use TaggerState.ProtoReflect.Descriptor instead. func (*TaggerState) Descriptor() ([]byte, []int) { - return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{13} + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{15} } func (x *TaggerState) GetState() map[string]*Entity { @@ -945,7 +1043,7 @@ type TaggerStateResponse struct { func (x *TaggerStateResponse) Reset() { *x = TaggerStateResponse{} - mi := &file_datadog_model_v1_model_proto_msgTypes[14] + mi := &file_datadog_model_v1_model_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -957,7 +1055,7 @@ func (x *TaggerStateResponse) String() string { func (*TaggerStateResponse) ProtoMessage() {} func (x *TaggerStateResponse) ProtoReflect() protoreflect.Message { - mi := &file_datadog_model_v1_model_proto_msgTypes[14] + mi := &file_datadog_model_v1_model_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -970,7 +1068,7 @@ func (x *TaggerStateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TaggerStateResponse.ProtoReflect.Descriptor instead. func (*TaggerStateResponse) Descriptor() ([]byte, []int) { - return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{14} + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{16} } func (x *TaggerStateResponse) GetLoaded() bool { @@ -980,6 +1078,138 @@ func (x *TaggerStateResponse) GetLoaded() bool { return false } +// Nested message for the local data +type GenerateContainerIDFromOriginInfoRequest_LocalData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ProcessID *uint32 `protobuf:"varint,1,opt,name=processID,proto3,oneof" json:"processID,omitempty"` // Process ID of the container process on the host. + ContainerID *string `protobuf:"bytes,2,opt,name=containerID,proto3,oneof" json:"containerID,omitempty"` // Container ID send from the client. + Inode *uint64 `protobuf:"varint,3,opt,name=inode,proto3,oneof" json:"inode,omitempty"` // Cgroup inode of the container. + PodUID *string `protobuf:"bytes,4,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID send from the client. +} + +func (x *GenerateContainerIDFromOriginInfoRequest_LocalData) Reset() { + *x = GenerateContainerIDFromOriginInfoRequest_LocalData{} + mi := &file_datadog_model_v1_model_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateContainerIDFromOriginInfoRequest_LocalData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateContainerIDFromOriginInfoRequest_LocalData) ProtoMessage() {} + +func (x *GenerateContainerIDFromOriginInfoRequest_LocalData) ProtoReflect() protoreflect.Message { + mi := &file_datadog_model_v1_model_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateContainerIDFromOriginInfoRequest_LocalData.ProtoReflect.Descriptor instead. +func (*GenerateContainerIDFromOriginInfoRequest_LocalData) Descriptor() ([]byte, []int) { + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{9, 0} +} + +func (x *GenerateContainerIDFromOriginInfoRequest_LocalData) GetProcessID() uint32 { + if x != nil && x.ProcessID != nil { + return *x.ProcessID + } + return 0 +} + +func (x *GenerateContainerIDFromOriginInfoRequest_LocalData) GetContainerID() string { + if x != nil && x.ContainerID != nil { + return *x.ContainerID + } + return "" +} + +func (x *GenerateContainerIDFromOriginInfoRequest_LocalData) GetInode() uint64 { + if x != nil && x.Inode != nil { + return *x.Inode + } + return 0 +} + +func (x *GenerateContainerIDFromOriginInfoRequest_LocalData) GetPodUID() string { + if x != nil && x.PodUID != nil { + return *x.PodUID + } + return "" +} + +// Nested message for the external data +type GenerateContainerIDFromOriginInfoRequest_ExternalData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Init *bool `protobuf:"varint,1,opt,name=init,proto3,oneof" json:"init,omitempty"` // Init is true if the container is an init container. + ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name in the Kubernetes Pod spec. + PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID in the Kubernetes Pod spec. +} + +func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) Reset() { + *x = GenerateContainerIDFromOriginInfoRequest_ExternalData{} + mi := &file_datadog_model_v1_model_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateContainerIDFromOriginInfoRequest_ExternalData) ProtoMessage() {} + +func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) ProtoReflect() protoreflect.Message { + mi := &file_datadog_model_v1_model_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateContainerIDFromOriginInfoRequest_ExternalData.ProtoReflect.Descriptor instead. +func (*GenerateContainerIDFromOriginInfoRequest_ExternalData) Descriptor() ([]byte, []int) { + return file_datadog_model_v1_model_proto_rawDescGZIP(), []int{9, 1} +} + +func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) GetInit() bool { + if x != nil && x.Init != nil { + return *x.Init + } + return false +} + +func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) GetContainerName() string { + if x != nil && x.ContainerName != nil { + return *x.ContainerName + } + return "" +} + +func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) GetPodUID() string { + if x != nil && x.PodUID != nil { + return *x.PodUID + } + return "" +} + var File_datadog_model_v1_model_proto protoreflect.FileDescriptor var file_datadog_model_v1_model_proto_rawDesc = []byte{ @@ -1055,70 +1285,115 @@ var file_datadog_model_v1_model_proto_rawDesc = []byte{ 0x43, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x54, 0x61, 0x67, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x54, 0x61, 0x67, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x54, - 0x61, 0x67, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x12, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x49, 0x64, 0x52, 0x02, 0x69, 0x64, 0x12, 0x42, 0x0a, 0x0b, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, - 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x64, 0x61, - 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, - 0x61, 0x67, 0x43, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0b, 0x63, - 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x22, 0x99, 0x01, 0x0a, 0x13, 0x46, - 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, - 0x31, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x49, 0x64, 0x52, 0x02, 0x69, 0x64, 0x12, 0x42, - 0x0a, 0x0b, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x43, 0x61, 0x72, 0x64, 0x69, 0x6e, - 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0b, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, - 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x34, 0x0a, 0x08, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x22, 0xc2, 0x01, 0x0a, - 0x10, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x6f, 0x67, 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x4d, 0x73, - 0x67, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, - 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, - 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x70, - 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x24, 0x0a, - 0x0d, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, 0x72, 0x79, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, 0x72, 0x79, 0x53, - 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, 0x72, 0x79, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, 0x72, - 0x79, 0x22, 0x9f, 0x02, 0x0a, 0x0b, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x3e, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x28, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x41, 0x0a, 0x06, 0x70, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x61, 0x67, 0x73, 0x22, 0xff, 0x04, 0x0a, 0x28, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x46, 0x72, 0x6f, 0x6d, 0x4f, + 0x72, 0x69, 0x67, 0x69, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x67, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x72, + 0x69, 0x67, 0x69, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x09, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x88, 0x01, 0x01, 0x12, 0x70, 0x0a, 0x0c, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x47, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x48, 0x01, 0x52, 0x0c, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x88, 0x01, 0x01, 0x1a, 0xc0, 0x01, 0x0a, 0x09, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x09, 0x70, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x09, + 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x44, 0x88, 0x01, 0x01, 0x12, 0x25, 0x0a, 0x0b, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x01, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, + 0x88, 0x01, 0x01, 0x12, 0x19, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x04, 0x48, 0x02, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x88, 0x01, 0x01, 0x12, 0x1b, + 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x55, 0x49, 0x44, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x03, + 0x52, 0x06, 0x70, 0x6f, 0x64, 0x55, 0x49, 0x44, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, + 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x44, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x69, 0x6e, + 0x6f, 0x64, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x70, 0x6f, 0x64, 0x55, 0x49, 0x44, 0x1a, 0x95, + 0x01, 0x0a, 0x0c, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x12, + 0x17, 0x0a, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, + 0x04, 0x69, 0x6e, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x01, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, + 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x55, 0x49, 0x44, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x06, 0x70, 0x6f, 0x64, 0x55, 0x49, 0x44, 0x88, 0x01, 0x01, + 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x69, 0x6e, 0x69, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, + 0x70, 0x6f, 0x64, 0x55, 0x49, 0x44, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x44, 0x61, 0x74, 0x61, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x44, 0x61, 0x74, 0x61, 0x22, 0x4d, 0x0a, 0x29, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x46, 0x72, 0x6f, 0x6d, + 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, + 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x49, 0x44, 0x22, 0x84, 0x01, 0x0a, 0x12, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, + 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, + 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x49, 0x64, 0x52, 0x02, 0x69, 0x64, 0x12, 0x42, 0x0a, 0x0b, 0x63, 0x61, 0x72, 0x64, 0x69, + 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, + 0x54, 0x61, 0x67, 0x43, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0b, + 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x22, 0x99, 0x01, 0x0a, 0x13, + 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x49, 0x64, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x42, 0x0a, 0x0b, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x43, 0x61, 0x72, 0x64, 0x69, + 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0b, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, + 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x34, 0x0a, 0x08, 0x45, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x10, 0x0a, 0x03, 0x75, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x22, 0xc2, 0x01, + 0x0a, 0x10, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x6f, 0x67, 0x73, 0x74, 0x61, 0x74, 0x73, 0x64, 0x4d, + 0x73, 0x67, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, + 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, + 0x70, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x24, + 0x0a, 0x0d, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, 0x72, 0x79, 0x53, 0x69, 0x7a, 0x65, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, 0x72, 0x79, + 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, 0x72, + 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x61, 0x6e, 0x63, 0x69, 0x6c, 0x6c, 0x61, + 0x72, 0x79, 0x22, 0x9f, 0x02, 0x0a, 0x0b, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x50, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x69, - 0x64, 0x4d, 0x61, 0x70, 0x1a, 0x52, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x69, 0x64, 0x4d, - 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x2d, 0x0a, 0x13, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, - 0x61, 0x64, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6c, 0x6f, 0x61, 0x64, - 0x65, 0x64, 0x2a, 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x09, 0x0a, 0x05, 0x41, 0x44, 0x44, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x4f, - 0x44, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, - 0x54, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x35, 0x0a, 0x0e, 0x54, 0x61, 0x67, 0x43, 0x61, 0x72, 0x64, - 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x4f, 0x57, 0x10, 0x00, - 0x12, 0x10, 0x0a, 0x0c, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, 0x54, 0x4f, 0x52, - 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x49, 0x47, 0x48, 0x10, 0x02, 0x42, 0x15, 0x5a, 0x13, - 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x67, 0x6f, 0x2f, 0x63, - 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x41, 0x0a, 0x06, 0x70, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x2e, 0x50, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, + 0x69, 0x64, 0x4d, 0x61, 0x70, 0x1a, 0x52, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x69, 0x64, + 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x2d, 0x0a, 0x13, 0x54, 0x61, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, + 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6c, 0x6f, 0x61, + 0x64, 0x65, 0x64, 0x2a, 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x09, 0x0a, 0x05, 0x41, 0x44, 0x44, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4d, + 0x4f, 0x44, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, + 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x35, 0x0a, 0x0e, 0x54, 0x61, 0x67, 0x43, 0x61, 0x72, + 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x4f, 0x57, 0x10, + 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, 0x54, 0x4f, + 0x52, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x49, 0x47, 0x48, 0x10, 0x02, 0x42, 0x15, 0x5a, + 0x13, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x67, 0x6f, 0x2f, + 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1134,27 +1409,31 @@ func file_datadog_model_v1_model_proto_rawDescGZIP() []byte { } var file_datadog_model_v1_model_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_datadog_model_v1_model_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_datadog_model_v1_model_proto_msgTypes = make([]protoimpl.MessageInfo, 21) var file_datadog_model_v1_model_proto_goTypes = []any{ - (EventType)(0), // 0: datadog.model.v1.EventType - (TagCardinality)(0), // 1: datadog.model.v1.TagCardinality - (*HostnameRequest)(nil), // 2: datadog.model.v1.HostnameRequest - (*HostnameReply)(nil), // 3: datadog.model.v1.HostnameReply - (*CaptureTriggerRequest)(nil), // 4: datadog.model.v1.CaptureTriggerRequest - (*CaptureTriggerResponse)(nil), // 5: datadog.model.v1.CaptureTriggerResponse - (*StreamTagsRequest)(nil), // 6: datadog.model.v1.StreamTagsRequest - (*StreamTagsResponse)(nil), // 7: datadog.model.v1.StreamTagsResponse - (*StreamTagsEvent)(nil), // 8: datadog.model.v1.StreamTagsEvent - (*DeprecatedFilter)(nil), // 9: datadog.model.v1.DeprecatedFilter - (*Entity)(nil), // 10: datadog.model.v1.Entity - (*FetchEntityRequest)(nil), // 11: datadog.model.v1.FetchEntityRequest - (*FetchEntityResponse)(nil), // 12: datadog.model.v1.FetchEntityResponse - (*EntityId)(nil), // 13: datadog.model.v1.EntityId - (*UnixDogstatsdMsg)(nil), // 14: datadog.model.v1.UnixDogstatsdMsg - (*TaggerState)(nil), // 15: datadog.model.v1.TaggerState - (*TaggerStateResponse)(nil), // 16: datadog.model.v1.TaggerStateResponse - nil, // 17: datadog.model.v1.TaggerState.StateEntry - nil, // 18: datadog.model.v1.TaggerState.PidMapEntry + (EventType)(0), // 0: datadog.model.v1.EventType + (TagCardinality)(0), // 1: datadog.model.v1.TagCardinality + (*HostnameRequest)(nil), // 2: datadog.model.v1.HostnameRequest + (*HostnameReply)(nil), // 3: datadog.model.v1.HostnameReply + (*CaptureTriggerRequest)(nil), // 4: datadog.model.v1.CaptureTriggerRequest + (*CaptureTriggerResponse)(nil), // 5: datadog.model.v1.CaptureTriggerResponse + (*StreamTagsRequest)(nil), // 6: datadog.model.v1.StreamTagsRequest + (*StreamTagsResponse)(nil), // 7: datadog.model.v1.StreamTagsResponse + (*StreamTagsEvent)(nil), // 8: datadog.model.v1.StreamTagsEvent + (*DeprecatedFilter)(nil), // 9: datadog.model.v1.DeprecatedFilter + (*Entity)(nil), // 10: datadog.model.v1.Entity + (*GenerateContainerIDFromOriginInfoRequest)(nil), // 11: datadog.model.v1.GenerateContainerIDFromOriginInfoRequest + (*GenerateContainerIDFromOriginInfoResponse)(nil), // 12: datadog.model.v1.GenerateContainerIDFromOriginInfoResponse + (*FetchEntityRequest)(nil), // 13: datadog.model.v1.FetchEntityRequest + (*FetchEntityResponse)(nil), // 14: datadog.model.v1.FetchEntityResponse + (*EntityId)(nil), // 15: datadog.model.v1.EntityId + (*UnixDogstatsdMsg)(nil), // 16: datadog.model.v1.UnixDogstatsdMsg + (*TaggerState)(nil), // 17: datadog.model.v1.TaggerState + (*TaggerStateResponse)(nil), // 18: datadog.model.v1.TaggerStateResponse + (*GenerateContainerIDFromOriginInfoRequest_LocalData)(nil), // 19: datadog.model.v1.GenerateContainerIDFromOriginInfoRequest.LocalData + (*GenerateContainerIDFromOriginInfoRequest_ExternalData)(nil), // 20: datadog.model.v1.GenerateContainerIDFromOriginInfoRequest.ExternalData + nil, // 21: datadog.model.v1.TaggerState.StateEntry + nil, // 22: datadog.model.v1.TaggerState.PidMapEntry } var file_datadog_model_v1_model_proto_depIdxs = []int32{ 1, // 0: datadog.model.v1.StreamTagsRequest.cardinality:type_name -> datadog.model.v1.TagCardinality @@ -1163,19 +1442,21 @@ var file_datadog_model_v1_model_proto_depIdxs = []int32{ 8, // 3: datadog.model.v1.StreamTagsResponse.events:type_name -> datadog.model.v1.StreamTagsEvent 0, // 4: datadog.model.v1.StreamTagsEvent.type:type_name -> datadog.model.v1.EventType 10, // 5: datadog.model.v1.StreamTagsEvent.entity:type_name -> datadog.model.v1.Entity - 13, // 6: datadog.model.v1.Entity.id:type_name -> datadog.model.v1.EntityId - 13, // 7: datadog.model.v1.FetchEntityRequest.id:type_name -> datadog.model.v1.EntityId - 1, // 8: datadog.model.v1.FetchEntityRequest.cardinality:type_name -> datadog.model.v1.TagCardinality - 13, // 9: datadog.model.v1.FetchEntityResponse.id:type_name -> datadog.model.v1.EntityId - 1, // 10: datadog.model.v1.FetchEntityResponse.cardinality:type_name -> datadog.model.v1.TagCardinality - 17, // 11: datadog.model.v1.TaggerState.state:type_name -> datadog.model.v1.TaggerState.StateEntry - 18, // 12: datadog.model.v1.TaggerState.pidMap:type_name -> datadog.model.v1.TaggerState.PidMapEntry - 10, // 13: datadog.model.v1.TaggerState.StateEntry.value:type_name -> datadog.model.v1.Entity - 14, // [14:14] is the sub-list for method output_type - 14, // [14:14] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 15, // 6: datadog.model.v1.Entity.id:type_name -> datadog.model.v1.EntityId + 19, // 7: datadog.model.v1.GenerateContainerIDFromOriginInfoRequest.localData:type_name -> datadog.model.v1.GenerateContainerIDFromOriginInfoRequest.LocalData + 20, // 8: datadog.model.v1.GenerateContainerIDFromOriginInfoRequest.externalData:type_name -> datadog.model.v1.GenerateContainerIDFromOriginInfoRequest.ExternalData + 15, // 9: datadog.model.v1.FetchEntityRequest.id:type_name -> datadog.model.v1.EntityId + 1, // 10: datadog.model.v1.FetchEntityRequest.cardinality:type_name -> datadog.model.v1.TagCardinality + 15, // 11: datadog.model.v1.FetchEntityResponse.id:type_name -> datadog.model.v1.EntityId + 1, // 12: datadog.model.v1.FetchEntityResponse.cardinality:type_name -> datadog.model.v1.TagCardinality + 21, // 13: datadog.model.v1.TaggerState.state:type_name -> datadog.model.v1.TaggerState.StateEntry + 22, // 14: datadog.model.v1.TaggerState.pidMap:type_name -> datadog.model.v1.TaggerState.PidMapEntry + 10, // 15: datadog.model.v1.TaggerState.StateEntry.value:type_name -> datadog.model.v1.Entity + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name } func init() { file_datadog_model_v1_model_proto_init() } @@ -1183,13 +1464,16 @@ func file_datadog_model_v1_model_proto_init() { if File_datadog_model_v1_model_proto != nil { return } + file_datadog_model_v1_model_proto_msgTypes[9].OneofWrappers = []any{} + file_datadog_model_v1_model_proto_msgTypes[17].OneofWrappers = []any{} + file_datadog_model_v1_model_proto_msgTypes[18].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_datadog_model_v1_model_proto_rawDesc, NumEnums: 2, - NumMessages: 17, + NumMessages: 21, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go b/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go index 7d7798b724aaa..a28d85a34a874 100644 --- a/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go +++ b/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go @@ -299,6 +299,26 @@ func (mr *MockAgentSecureClientMockRecorder) TaggerFetchEntity(ctx, in interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TaggerFetchEntity", reflect.TypeOf((*MockAgentSecureClient)(nil).TaggerFetchEntity), varargs...) } +// TaggerGenerateContainerIDFromOriginInfo mocks base method. +func (m *MockAgentSecureClient) TaggerGenerateContainerIDFromOriginInfo(ctx context.Context, in *core.GenerateContainerIDFromOriginInfoRequest, opts ...grpc.CallOption) (*core.GenerateContainerIDFromOriginInfoResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "TaggerGenerateContainerIDFromOriginInfo", varargs...) + ret0, _ := ret[0].(*core.GenerateContainerIDFromOriginInfoResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TaggerGenerateContainerIDFromOriginInfo indicates an expected call of TaggerGenerateContainerIDFromOriginInfo. +func (mr *MockAgentSecureClientMockRecorder) TaggerGenerateContainerIDFromOriginInfo(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TaggerGenerateContainerIDFromOriginInfo", reflect.TypeOf((*MockAgentSecureClient)(nil).TaggerGenerateContainerIDFromOriginInfo), varargs...) +} + // TaggerStreamEntities mocks base method. func (m *MockAgentSecureClient) TaggerStreamEntities(ctx context.Context, in *core.StreamTagsRequest, opts ...grpc.CallOption) (core.AgentSecure_TaggerStreamEntitiesClient, error) { m.ctrl.T.Helper() @@ -865,6 +885,21 @@ func (mr *MockAgentSecureServerMockRecorder) TaggerFetchEntity(arg0, arg1 interf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TaggerFetchEntity", reflect.TypeOf((*MockAgentSecureServer)(nil).TaggerFetchEntity), arg0, arg1) } +// TaggerGenerateContainerIDFromOriginInfo mocks base method. +func (m *MockAgentSecureServer) TaggerGenerateContainerIDFromOriginInfo(arg0 context.Context, arg1 *core.GenerateContainerIDFromOriginInfoRequest) (*core.GenerateContainerIDFromOriginInfoResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TaggerGenerateContainerIDFromOriginInfo", arg0, arg1) + ret0, _ := ret[0].(*core.GenerateContainerIDFromOriginInfoResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TaggerGenerateContainerIDFromOriginInfo indicates an expected call of TaggerGenerateContainerIDFromOriginInfo. +func (mr *MockAgentSecureServerMockRecorder) TaggerGenerateContainerIDFromOriginInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TaggerGenerateContainerIDFromOriginInfo", reflect.TypeOf((*MockAgentSecureServer)(nil).TaggerGenerateContainerIDFromOriginInfo), arg0, arg1) +} + // TaggerStreamEntities mocks base method. func (m *MockAgentSecureServer) TaggerStreamEntities(arg0 *core.StreamTagsRequest, arg1 core.AgentSecure_TaggerStreamEntitiesServer) error { m.ctrl.T.Helper() diff --git a/pkg/security/agent/agent_nix.go b/pkg/security/agent/agent_nix.go index bc444d6f049fe..8083e3259b87e 100644 --- a/pkg/security/agent/agent_nix.go +++ b/pkg/security/agent/agent_nix.go @@ -12,9 +12,10 @@ import ( "go.uber.org/atomic" + "github.com/DataDog/datadog-go/v5/statsd" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/security/security_profile/dump" - "github.com/DataDog/datadog-go/v5/statsd" ) // NewRuntimeSecurityAgent instantiates a new RuntimeSecurityAgent @@ -30,7 +31,7 @@ func NewRuntimeSecurityAgent(statsdClient statsd.ClientInterface, hostname strin } // on windows do no storage manager - storage, err := dump.NewAgentStorageManager() + storage, err := dump.NewAgentStorageManager(statsdClient) if err != nil { return nil, err } diff --git a/pkg/security/config/config.go b/pkg/security/config/config.go index ab6b0b6b591e1..9d165a4ae1900 100644 --- a/pkg/security/config/config.go +++ b/pkg/security/config/config.go @@ -101,6 +101,9 @@ type RuntimeSecurityConfig struct { // ActivityDumpTracedCgroupsCount defines the maximum count of cgroups that should be monitored concurrently. Leave this parameter to 0 to prevent the generation // of activity dumps based on cgroups. ActivityDumpTracedCgroupsCount int + // ActivityDumpCgroupsManagers defines the cgroup managers we generate dumps for. + ActivityDumpCgroupsManagers []string + // ActivityDumpTracedEventTypes defines the list of events that should be captured in an activity dump. Leave this // parameter empty to monitor all event types. If not already present, the `exec` event will automatically be added // to this list. @@ -368,6 +371,7 @@ func NewRuntimeSecurityConfig() (*RuntimeSecurityConfig, error) { ActivityDumpLoadControlPeriod: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.load_controller_period"), ActivityDumpLoadControlMinDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.min_timeout"), ActivityDumpTracedCgroupsCount: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.traced_cgroups_count"), + ActivityDumpCgroupsManagers: pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.cgroup_managers"), ActivityDumpTracedEventTypes: parseEventTypeStringSlice(pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.traced_event_types")), ActivityDumpCgroupDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.dump_duration"), ActivityDumpRateLimiter: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.rate_limiter"), diff --git a/pkg/security/ebpf/c/include/constants/custom.h b/pkg/security/ebpf/c/include/constants/custom.h index 88be17fa3c80b..1e2311bf8fc7e 100644 --- a/pkg/security/ebpf/c/include/constants/custom.h +++ b/pkg/security/ebpf/c/include/constants/custom.h @@ -196,4 +196,8 @@ static __attribute__((always_inline)) u64 get_imds_ip() { #define CGROUP_MANAGER_CRI 4 #define CGROUP_MANAGER_SYSTEMD 5 +#define CGROUP_MANAGER_MASK 0b111 +#define CGROUP_SYSTEMD_SERVICE (0 << 8) +#define CGROUP_SYSTEMD_SCOPE (1 << 8) + #endif diff --git a/pkg/security/ebpf/c/include/constants/offsets/filesystem.h b/pkg/security/ebpf/c/include/constants/offsets/filesystem.h index 24b2d5f769af1..797070a8534e7 100644 --- a/pkg/security/ebpf/c/include/constants/offsets/filesystem.h +++ b/pkg/security/ebpf/c/include/constants/offsets/filesystem.h @@ -9,8 +9,11 @@ struct mount; unsigned long __attribute__((always_inline)) get_inode_ino(struct inode *inode) { + u64 inode_ino_offset; + LOAD_CONSTANT("inode_ino_offset", inode_ino_offset); + unsigned long ino; - bpf_probe_read(&ino, sizeof(inode), &inode->i_ino); + bpf_probe_read(&ino, sizeof(inode), (void *)inode + inode_ino_offset); return ino; } diff --git a/pkg/security/ebpf/c/include/helpers/activity_dump.h b/pkg/security/ebpf/c/include/helpers/activity_dump.h index c0f8d246006bf..70e776cca8161 100644 --- a/pkg/security/ebpf/c/include/helpers/activity_dump.h +++ b/pkg/security/ebpf/c/include/helpers/activity_dump.h @@ -53,10 +53,16 @@ __attribute__((always_inline)) struct cgroup_tracing_event_t *get_cgroup_tracing return evt; } +__attribute__((always_inline)) u32 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { + u32 cgroup_manager = cgroup->cgroup_flags & CGROUP_MANAGER_MASK; + u32 supported = (cgroup->cgroup_flags != 0) && (bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_manager) != NULL); + return supported; +} + __attribute__((always_inline)) bool reserve_traced_cgroup_spot(struct cgroup_context_t *cgroup, u64 now, u64 cookie, struct activity_dump_config *config) { // insert dump config defaults - u32 defaults_key = 0; - struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &defaults_key); + u32 cgroup_flags = cgroup->cgroup_flags; + struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_flags); if (defaults == NULL) { // should never happen, ignore return false; @@ -102,11 +108,15 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return 0; } - if ((container->cgroup_context.cgroup_flags & 0b111) == CGROUP_MANAGER_SYSTEMD) { + if (!is_cgroup_activity_dumps_supported(&container->cgroup_context)) { return 0; } - copy_container_id(container->container_id, evt->container.container_id); + if ((container->cgroup_context.cgroup_flags&CGROUP_MANAGER_MASK) != CGROUP_MANAGER_SYSTEMD) { + copy_container_id(container->container_id, evt->container.container_id); + } else { + evt->container.container_id[0] = '\0'; + } evt->container.cgroup_context = container->cgroup_context; evt->cookie = cookie; evt->config = config; @@ -115,10 +125,6 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return cookie; } -__attribute__((always_inline)) u64 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { - return (cgroup->cgroup_flags != 0) && ((cgroup->cgroup_flags&0b111) != CGROUP_MANAGER_SYSTEMD); -} - __attribute__((always_inline)) u64 should_trace_new_process_cgroup(void *ctx, u64 now, u32 pid, struct container_context_t *container) { // should we start tracing this cgroup ? struct cgroup_context_t cgroup_context; diff --git a/pkg/security/ebpf/c/include/helpers/filesystem.h b/pkg/security/ebpf/c/include/helpers/filesystem.h index 07ca0431d7382..94210013c123e 100644 --- a/pkg/security/ebpf/c/include/helpers/filesystem.h +++ b/pkg/security/ebpf/c/include/helpers/filesystem.h @@ -102,10 +102,15 @@ void __attribute__((always_inline)) fill_file(struct dentry *dentry, struct file file->dev = get_dentry_dev(dentry); - bpf_probe_read(&file->metadata.nlink, sizeof(file->metadata.nlink), (void *)&d_inode->i_nlink); + u64 inode_nlink_offset; + LOAD_CONSTANT("inode_nlink_offset", inode_nlink_offset); + u64 inode_gid_offset; + LOAD_CONSTANT("inode_gid_offset", inode_gid_offset); + + bpf_probe_read(&file->metadata.nlink, sizeof(file->metadata.nlink), (void *)d_inode + inode_nlink_offset); bpf_probe_read(&file->metadata.mode, sizeof(file->metadata.mode), &d_inode->i_mode); bpf_probe_read(&file->metadata.uid, sizeof(file->metadata.uid), &d_inode->i_uid); - bpf_probe_read(&file->metadata.gid, sizeof(file->metadata.gid), &d_inode->i_gid); + bpf_probe_read(&file->metadata.gid, sizeof(file->metadata.gid), (void *)d_inode + inode_gid_offset); u64 inode_ctime_sec_offset; LOAD_CONSTANT("inode_ctime_sec_offset", inode_ctime_sec_offset); @@ -118,10 +123,10 @@ void __attribute__((always_inline)) fill_file(struct dentry *dentry, struct file bpf_probe_read(&nsec, sizeof(nsec), (void *)d_inode + inode_ctime_nsec_offset); file->metadata.ctime.tv_nsec = nsec; } else { -#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0) - bpf_probe_read(&file->metadata.ctime, sizeof(file->metadata.ctime), &d_inode->i_ctime); -#elif LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) - bpf_probe_read(&file->metadata.ctime, sizeof(file->metadata.ctime), &d_inode->__i_ctime); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) + u64 inode_ctime_offset; + LOAD_CONSTANT("inode_ctime_offset", inode_ctime_offset); + bpf_probe_read(&file->metadata.ctime, sizeof(file->metadata.ctime), (void *)d_inode + inode_ctime_offset); #else bpf_probe_read(&file->metadata.ctime.tv_sec, sizeof(file->metadata.ctime.tv_sec), &d_inode->i_ctime_sec); bpf_probe_read(&file->metadata.ctime.tv_nsec, sizeof(file->metadata.ctime.tv_nsec), &d_inode->i_ctime_nsec); @@ -139,10 +144,10 @@ void __attribute__((always_inline)) fill_file(struct dentry *dentry, struct file bpf_probe_read(&nsec, sizeof(nsec), (void *)d_inode + inode_mtime_nsec_offset); file->metadata.mtime.tv_nsec = nsec; } else { -#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0) - bpf_probe_read(&file->metadata.mtime, sizeof(file->metadata.mtime), &d_inode->i_mtime); -#elif LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) - bpf_probe_read(&file->metadata.mtime, sizeof(file->metadata.mtime), &d_inode->__i_mtime); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) + u64 inode_mtime_offset; + LOAD_CONSTANT("inode_mtime_offset", inode_mtime_offset); + bpf_probe_read(&file->metadata.mtime, sizeof(file->metadata.mtime), (void *)d_inode + inode_mtime_offset); #else bpf_probe_read(&file->metadata.mtime.tv_sec, sizeof(file->metadata.mtime.tv_sec), &d_inode->i_mtime_sec); bpf_probe_read(&file->metadata.mtime.tv_nsec, sizeof(file->metadata.mtime.tv_nsec), &d_inode->i_mtime_nsec); diff --git a/pkg/security/ebpf/c/include/hooks/cgroup.h b/pkg/security/ebpf/c/include/hooks/cgroup.h index a231142b8f90a..b7ce66a870b92 100644 --- a/pkg/security/ebpf/c/include/hooks/cgroup.h +++ b/pkg/security/ebpf/c/include/hooks/cgroup.h @@ -189,13 +189,13 @@ static __attribute__((always_inline)) int trace__cgroup_write(ctx_t *ctx) { #endif int length = bpf_probe_read_str(prefix, sizeof(cgroup_prefix_t), container_id) & 0xff; - if (cgroup_flags == 0 && ( - (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') - || - (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') - )) { - cgroup_flags = CGROUP_MANAGER_SYSTEMD; - } else if (cgroup_flags != 0) { + if (cgroup_flags == 0) { + if (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SERVICE; + } else if (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SCOPE; + } + } else { bpf_probe_read(&new_entry.container.container_id, sizeof(new_entry.container.container_id), container_id); } diff --git a/pkg/security/ebpf/c/include/maps.h b/pkg/security/ebpf/c/include/maps.h index d6ab3ceb74dfb..3b821b19fe5cf 100644 --- a/pkg/security/ebpf/c/include/maps.h +++ b/pkg/security/ebpf/c/include/maps.h @@ -28,7 +28,7 @@ BPF_ARRAY_MAP(syscall_ctx_gen_id, u32, 1) BPF_ARRAY_MAP(syscall_ctx, char[MAX_SYSCALL_CTX_SIZE], MAX_SYSCALL_CTX_ENTRIES) BPF_HASH_MAP(activity_dumps_config, u64, struct activity_dump_config, 1) // max entries will be overridden at runtime -BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 1) +BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 5) BPF_HASH_MAP(traced_cgroups, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(cgroup_wait_list, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(traced_pids, u32, u64, 8192) // max entries will be overridden at runtime diff --git a/pkg/security/ebpf/kernel/kernel.go b/pkg/security/ebpf/kernel/kernel.go index 7ff8ef846ac9e..646163b104936 100644 --- a/pkg/security/ebpf/kernel/kernel.go +++ b/pkg/security/ebpf/kernel/kernel.go @@ -103,6 +103,8 @@ var ( Kernel6_5 = kernel.VersionCode(6, 5, 0) // Kernel6_6 is the KernelVersion representation of kernel version 6.6 Kernel6_6 = kernel.VersionCode(6, 6, 0) + // Kernel6_7 is the KernelVersion representation of kernel version 6.7 + Kernel6_7 = kernel.VersionCode(6, 7, 0) // Kernel6_10 is the KernelVersion representation of kernel version 6.10 Kernel6_10 = kernel.VersionCode(6, 10, 0) // Kernel6_11 is the KernelVersion representation of kernel version 6.11 diff --git a/pkg/security/probe/config/config.go b/pkg/security/probe/config/config.go index c312bbdea3181..c597eebe84e0e 100644 --- a/pkg/security/probe/config/config.go +++ b/pkg/security/probe/config/config.go @@ -101,12 +101,6 @@ type Config struct { // RuntimeCompilationEnabled defines if the runtime-compilation is enabled RuntimeCompilationEnabled bool - // EnableRuntimeCompiledConstants defines if the runtime compilation based constant fetcher is enabled - RuntimeCompiledConstantsEnabled bool - - // RuntimeCompiledConstantsIsSet is set if the runtime compiled constants option is user-set - RuntimeCompiledConstantsIsSet bool - // NetworkLazyInterfacePrefixes is the list of interfaces prefix that aren't explicitly deleted by the container // runtime, and that are lazily deleted by the kernel when a network namespace is cleaned up. This list helps the // agent detect when a network namespace should be purged from all caches. @@ -190,9 +184,7 @@ func NewConfig() (*Config, error) { EventServerBurst: pkgconfigsetup.SystemProbe().GetInt(join(evNS, "event_server.burst")), // runtime compilation - RuntimeCompilationEnabled: getBool("runtime_compilation.enabled"), - RuntimeCompiledConstantsEnabled: getBool("runtime_compilation.compiled_constants_enabled"), - RuntimeCompiledConstantsIsSet: isSet("runtime_compilation.compiled_constants_enabled"), + RuntimeCompilationEnabled: getBool("runtime_compilation.enabled"), } if err := c.sanitize(); err != nil { @@ -225,10 +217,6 @@ func (c *Config) sanitize() error { c.RuntimeCompilationEnabled = false } - if !c.RuntimeCompilationEnabled { - c.RuntimeCompiledConstantsEnabled = false - } - if c.EventStreamBufferSize%os.Getpagesize() != 0 || c.EventStreamBufferSize&(c.EventStreamBufferSize-1) != 0 { return fmt.Errorf("runtime_security_config.event_stream.buffer_size must be a power of 2 and a multiple of %d", os.Getpagesize()) } diff --git a/pkg/security/probe/constantfetch/available.go b/pkg/security/probe/constantfetch/available.go index bdbb9edcf4620..1035fb497e600 100644 --- a/pkg/security/probe/constantfetch/available.go +++ b/pkg/security/probe/constantfetch/available.go @@ -12,7 +12,6 @@ import ( "errors" "fmt" - "github.com/DataDog/datadog-go/v5/statsd" "github.com/cilium/ebpf/btf" pkgebpf "github.com/DataDog/datadog-agent/pkg/ebpf" @@ -22,18 +21,13 @@ import ( ) // GetAvailableConstantFetchers returns available constant fetchers -func GetAvailableConstantFetchers(config *config.Config, kv *kernel.Version, statsdClient statsd.ClientInterface) []ConstantFetcher { +func GetAvailableConstantFetchers(config *config.Config, kv *kernel.Version) []ConstantFetcher { fetchers := make([]ConstantFetcher, 0) if coreFetcher, err := NewBTFConstantFetcherFromCurrentKernel(); err == nil { fetchers = append(fetchers, coreFetcher) } - if config.RuntimeCompiledConstantsEnabled { - rcConstantFetcher := NewRuntimeCompilationConstantFetcher(&config.Config, statsdClient) - fetchers = append(fetchers, rcConstantFetcher) - } - btfhubFetcher, err := NewBTFHubConstantFetcher(kv) if err != nil { seclog.Debugf("failed to create btfhub constant fetcher: %v", err) diff --git a/pkg/security/probe/constantfetch/available_unsupported.go b/pkg/security/probe/constantfetch/available_unsupported.go index 3e003cbc28601..6195452074bbc 100644 --- a/pkg/security/probe/constantfetch/available_unsupported.go +++ b/pkg/security/probe/constantfetch/available_unsupported.go @@ -11,15 +11,13 @@ package constantfetch import ( "errors" - "github.com/DataDog/datadog-go/v5/statsd" - "github.com/DataDog/datadog-agent/pkg/security/ebpf/kernel" "github.com/DataDog/datadog-agent/pkg/security/probe/config" "github.com/DataDog/datadog-agent/pkg/security/seclog" ) // GetAvailableConstantFetchers returns available constant fetchers -func GetAvailableConstantFetchers(_ *config.Config, kv *kernel.Version, _ statsd.ClientInterface) []ConstantFetcher { +func GetAvailableConstantFetchers(_ *config.Config, kv *kernel.Version) []ConstantFetcher { fetchers := make([]ConstantFetcher, 0) btfhubFetcher, err := NewBTFHubConstantFetcher(kv) diff --git a/pkg/security/probe/constantfetch/btfhub.go b/pkg/security/probe/constantfetch/btfhub.go index c81a2ee968153..1d7aaaa191454 100644 --- a/pkg/security/probe/constantfetch/btfhub.go +++ b/pkg/security/probe/constantfetch/btfhub.go @@ -79,12 +79,12 @@ func (f *BTFHubConstantFetcher) appendRequest(id string) { } // AppendSizeofRequest appends a sizeof request -func (f *BTFHubConstantFetcher) AppendSizeofRequest(id, _, _ string) { +func (f *BTFHubConstantFetcher) AppendSizeofRequest(id, _ string) { f.appendRequest(id) } // AppendOffsetofRequest appends an offset request -func (f *BTFHubConstantFetcher) AppendOffsetofRequest(id, _, _, _ string) { +func (f *BTFHubConstantFetcher) AppendOffsetofRequest(id, _, _ string) { f.appendRequest(id) } diff --git a/pkg/security/probe/constantfetch/btfhub/constants.json b/pkg/security/probe/constantfetch/btfhub/constants.json index abf445a40119a..90238609bab82 100644 --- a/pkg/security/probe/constantfetch/btfhub/constants.json +++ b/pkg/security/probe/constantfetch/btfhub/constants.json @@ -20,6 +20,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -66,6 +71,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -112,6 +122,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -158,6 +173,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -204,6 +224,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -250,6 +275,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -296,6 +326,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -342,6 +377,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -388,6 +428,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -434,6 +479,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -477,6 +527,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -520,6 +575,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -566,6 +626,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -612,6 +677,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -658,6 +728,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -704,6 +779,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -750,6 +830,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -792,6 +877,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -833,6 +923,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -873,6 +968,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -915,6 +1015,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -959,6 +1064,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1003,6 +1113,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1050,6 +1165,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1095,6 +1215,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1140,6 +1265,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1185,6 +1315,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1230,6 +1365,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1277,6 +1417,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1323,6 +1468,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1363,6 +1513,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1404,6 +1559,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1451,6 +1611,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -1498,6 +1663,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -1545,6 +1715,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1592,6 +1767,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1640,6 +1820,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1686,6 +1871,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1732,6 +1922,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -1780,6 +1975,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1827,6 +2027,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1876,6 +2081,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1925,6 +2135,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1973,6 +2188,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2021,6 +2241,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2069,6 +2294,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2117,6 +2347,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2165,6 +2400,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2213,6 +2453,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2261,6 +2506,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2309,6 +2559,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2357,6 +2612,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2406,6 +2666,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2455,6 +2720,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2504,6 +2774,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2553,6 +2828,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2601,6 +2881,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2649,6 +2934,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2697,6 +2987,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2745,6 +3040,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2793,6 +3093,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2841,6 +3146,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2889,6 +3199,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2937,6 +3252,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2985,6 +3305,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3034,6 +3359,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3083,6 +3413,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3132,6 +3467,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3175,6 +3515,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3218,6 +3563,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3261,6 +3611,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3305,6 +3660,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3348,6 +3708,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3394,6 +3759,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3437,6 +3807,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3481,6 +3856,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3529,6 +3909,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3578,6 +3963,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3626,6 +4016,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3675,6 +4070,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -3722,6 +4122,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3771,6 +4176,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -3819,6 +4229,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3868,6 +4283,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -3918,6 +4338,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -3968,6 +4393,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4017,6 +4447,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4066,6 +4501,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4116,6 +4556,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4163,6 +4608,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4212,6 +4662,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4262,6 +4717,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4312,6 +4772,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4359,6 +4824,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4401,6 +4871,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4447,6 +4922,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4493,6 +4973,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4542,6 +5027,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4592,6 +5082,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4642,6 +5137,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4692,6 +5192,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4742,6 +5247,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4789,6 +5299,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4838,6 +5353,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4888,6 +5408,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4935,6 +5460,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4984,6 +5514,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5031,6 +5566,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5077,6 +5617,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5123,6 +5668,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5170,6 +5720,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5217,6 +5772,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5264,6 +5824,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5313,6 +5878,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5363,6 +5933,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5413,6 +5988,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5463,6 +6043,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5513,6 +6098,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5563,6 +6153,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5608,6 +6203,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5652,6 +6252,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5698,6 +6303,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5745,6 +6355,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5793,6 +6408,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -5840,6 +6460,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -5889,6 +6514,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -5932,6 +6562,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5975,6 +6610,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6021,6 +6661,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6067,6 +6712,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6114,6 +6764,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6160,6 +6815,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6207,6 +6867,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6256,6 +6921,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6306,6 +6976,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6356,6 +7031,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6406,6 +7086,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6451,6 +7136,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6495,6 +7185,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6539,6 +7234,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6583,6 +7283,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6629,6 +7334,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6677,6 +7387,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6725,6 +7440,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6773,6 +7493,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6821,6 +7546,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6869,6 +7599,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6917,6 +7652,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6965,6 +7705,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7014,6 +7759,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7057,6 +7807,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7100,6 +7855,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7143,6 +7903,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7186,6 +7951,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7229,6 +7999,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7272,6 +8047,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7315,6 +8095,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7358,6 +8143,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7401,6 +8191,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7444,6 +8239,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7487,6 +8287,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7530,6 +8335,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7573,6 +8383,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7616,6 +8431,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7659,6 +8479,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7702,6 +8527,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7745,6 +8575,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7791,6 +8626,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7839,6 +8679,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7887,6 +8732,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7936,6 +8786,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7985,6 +8840,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8033,6 +8893,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8081,6 +8946,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8129,6 +8999,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8179,6 +9054,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8229,6 +9109,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8279,6 +9164,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8329,6 +9219,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8376,6 +9271,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8424,6 +9324,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8472,6 +9377,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8520,6 +9430,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8568,6 +9483,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8616,6 +9536,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8664,6 +9589,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8713,6 +9643,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8762,6 +9697,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8811,6 +9751,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8860,6 +9805,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8908,6 +9858,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8956,6 +9911,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -9004,6 +9964,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -9052,6 +10017,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -9100,6 +10070,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9150,6 +10125,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9200,6 +10180,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9250,6 +10235,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9300,6 +10290,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9350,6 +10345,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9400,6 +10400,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9450,6 +10455,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9500,6 +10510,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9550,6 +10565,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9600,6 +10620,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9650,6 +10675,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9700,6 +10730,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9750,6 +10785,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -9799,6 +10839,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -9848,6 +10893,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9898,6 +10948,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9948,6 +11003,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9998,6 +11058,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10048,6 +11113,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10098,6 +11168,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10147,6 +11222,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10196,6 +11276,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10245,6 +11330,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10294,6 +11384,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10343,6 +11438,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10393,6 +11493,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10443,6 +11548,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10493,6 +11603,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10543,6 +11658,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10593,6 +11713,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10643,6 +11768,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10693,6 +11823,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10743,6 +11878,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10793,6 +11933,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10843,6 +11988,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10893,6 +12043,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10943,6 +12098,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10993,6 +12153,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -11043,6 +12208,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -11093,6 +12263,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11142,6 +12317,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11191,6 +12371,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11240,6 +12425,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11289,6 +12479,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11338,6 +12533,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, diff --git a/pkg/security/probe/constantfetch/constant_names.go b/pkg/security/probe/constantfetch/constant_names.go index 2feab9c5e0402..80d8b57c04a25 100644 --- a/pkg/security/probe/constantfetch/constant_names.go +++ b/pkg/security/probe/constantfetch/constant_names.go @@ -31,6 +31,13 @@ const ( OffsetNameDentryDSb = "dentry_d_sb_offset" OffsetNameMountMntID = "mount_id_offset" + // inode + OffsetInodeIno = "inode_ino_offset" + OffsetInodeNlink = "inode_nlink_offset" + OffsetInodeGid = "inode_gid_offset" + OffsetInodeMtime = "inode_mtime_offset" + OffsetInodeCtime = "inode_ctime_offset" + // inode times OffsetNameInodeCtimeSec = "inode_ctime_sec_offset" OffsetNameInodeCtimeNsec = "inode_ctime_nsec_offset" diff --git a/pkg/security/probe/constantfetch/core.go b/pkg/security/probe/constantfetch/core.go index eba5beaea450f..117fb9cf6187d 100644 --- a/pkg/security/probe/constantfetch/core.go +++ b/pkg/security/probe/constantfetch/core.go @@ -89,7 +89,7 @@ func (f *BTFConstantFetcher) runRequest(r constantRequest) { } // AppendSizeofRequest appends a sizeof request -func (f *BTFConstantFetcher) AppendSizeofRequest(id, typeName, _ string) { +func (f *BTFConstantFetcher) AppendSizeofRequest(id, typeName string) { f.runRequest(constantRequest{ id: id, sizeof: true, @@ -98,7 +98,7 @@ func (f *BTFConstantFetcher) AppendSizeofRequest(id, typeName, _ string) { } // AppendOffsetofRequest appends an offset request -func (f *BTFConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName, _ string) { +func (f *BTFConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName string) { f.runRequest(constantRequest{ id: id, sizeof: false, diff --git a/pkg/security/probe/constantfetch/fallback.go b/pkg/security/probe/constantfetch/fallback.go index 531edb59c29d3..c6ecea9a22c3e 100644 --- a/pkg/security/probe/constantfetch/fallback.go +++ b/pkg/security/probe/constantfetch/fallback.go @@ -147,17 +147,27 @@ func (f *FallbackConstantFetcher) appendRequest(id string) { value = getRenameStructOldDentryOffset(f.kernelVersion) case OffsetNameRenameStructNewDentry: value = getRenameStructNewDentryOffset(f.kernelVersion) + case OffsetInodeIno: + value = getInodeInoOffset(f.kernelVersion) + case OffsetInodeGid: + value = getInodeGIDOffset(f.kernelVersion) + case OffsetInodeNlink: + value = getInodeNlinkOffset(f.kernelVersion) + case OffsetInodeMtime: + value = getInodeMtimeOffset(f.kernelVersion) + case OffsetInodeCtime: + value = getInodeCtimeOffset(f.kernelVersion) } f.res[id] = value } // AppendSizeofRequest appends a sizeof request -func (f *FallbackConstantFetcher) AppendSizeofRequest(id, _, _ string) { +func (f *FallbackConstantFetcher) AppendSizeofRequest(id, _ string) { f.appendRequest(id) } // AppendOffsetofRequest appends an offset request -func (f *FallbackConstantFetcher) AppendOffsetofRequest(id, _, _, _ string) { +func (f *FallbackConstantFetcher) AppendOffsetofRequest(id, _, _ string) { f.appendRequest(id) } @@ -236,6 +246,26 @@ func getSizeOfStructInode(kv *kernel.Version) uint64 { return sizeOf } +func getInodeInoOffset(_ *kernel.Version) uint64 { + return uint64(64) +} + +func getInodeGIDOffset(_ *kernel.Version) uint64 { + return uint64(8) +} + +func getInodeNlinkOffset(_ *kernel.Version) uint64 { + return uint64(72) +} + +func getInodeMtimeOffset(_ *kernel.Version) uint64 { + return uint64(104) +} + +func getInodeCtimeOffset(_ *kernel.Version) uint64 { + return uint64(120) +} + func getSuperBlockFlagsOffset(_ *kernel.Version) uint64 { return uint64(80) } diff --git a/pkg/security/probe/constantfetch/fetcher.go b/pkg/security/probe/constantfetch/fetcher.go index cfde02df17e2f..8d03030facbd0 100644 --- a/pkg/security/probe/constantfetch/fetcher.go +++ b/pkg/security/probe/constantfetch/fetcher.go @@ -26,8 +26,8 @@ const ErrorSentinel uint64 = ^uint64(0) // eBPF relocations type ConstantFetcher interface { fmt.Stringer - AppendSizeofRequest(id, typeName, headerName string) - AppendOffsetofRequest(id, typeName, fieldName, headerName string) + AppendSizeofRequest(id, typeName string) + AppendOffsetofRequest(id, typeName, fieldName string) FinishAndGetResults() (map[string]uint64, error) } @@ -57,30 +57,27 @@ func (f *ComposeConstantFetcher) appendRequest(req *composeRequest) { _, _ = io.WriteString(f.hasher, req.id) _, _ = io.WriteString(f.hasher, req.typeName) _, _ = io.WriteString(f.hasher, req.fieldName) - _, _ = io.WriteString(f.hasher, req.headerName) } // AppendSizeofRequest appends a sizeof request -func (f *ComposeConstantFetcher) AppendSizeofRequest(id, typeName, headerName string) { +func (f *ComposeConstantFetcher) AppendSizeofRequest(id, typeName string) { f.appendRequest(&composeRequest{ - id: id, - sizeof: true, - typeName: typeName, - fieldName: "", - headerName: headerName, - value: ErrorSentinel, + id: id, + sizeof: true, + typeName: typeName, + fieldName: "", + value: ErrorSentinel, }) } // AppendOffsetofRequest appends an offset request -func (f *ComposeConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName, headerName string) { +func (f *ComposeConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName string) { f.appendRequest(&composeRequest{ - id: id, - sizeof: false, - typeName: typeName, - fieldName: fieldName, - headerName: headerName, - value: ErrorSentinel, + id: id, + sizeof: false, + typeName: typeName, + fieldName: fieldName, + value: ErrorSentinel, }) } @@ -98,9 +95,9 @@ func (f *ComposeConstantFetcher) fillConstantCacheIfNeeded() { for _, req := range f.requests { if req.value == ErrorSentinel { if req.sizeof { - fetcher.AppendSizeofRequest(req.id, req.typeName, req.headerName) + fetcher.AppendSizeofRequest(req.id, req.typeName) } else { - fetcher.AppendOffsetofRequest(req.id, req.typeName, req.fieldName, req.headerName) + fetcher.AppendOffsetofRequest(req.id, req.typeName, req.fieldName) } } } @@ -166,7 +163,6 @@ type composeRequest struct { id string sizeof bool typeName, fieldName string - headerName string value uint64 fetcherName string } diff --git a/pkg/security/probe/constantfetch/offset_guesser.go b/pkg/security/probe/constantfetch/offset_guesser.go index 903aa95b81a52..ad2dbb09cfe0e 100644 --- a/pkg/security/probe/constantfetch/offset_guesser.go +++ b/pkg/security/probe/constantfetch/offset_guesser.go @@ -160,11 +160,11 @@ func (og *OffsetGuesser) guess(id string) error { } // AppendSizeofRequest appends a sizeof request -func (og *OffsetGuesser) AppendSizeofRequest(_, _, _ string) { +func (og *OffsetGuesser) AppendSizeofRequest(_, _ string) { } // AppendOffsetofRequest appends an offset request -func (og *OffsetGuesser) AppendOffsetofRequest(id, _, _, _ string) { +func (og *OffsetGuesser) AppendOffsetofRequest(id, _, _ string) { og.res[id] = ErrorSentinel } diff --git a/pkg/security/probe/constantfetch/runtime_compiled.go b/pkg/security/probe/constantfetch/runtime_compiled.go deleted file mode 100644 index c847ab821e9a4..0000000000000 --- a/pkg/security/probe/constantfetch/runtime_compiled.go +++ /dev/null @@ -1,178 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux && linux_bpf - -// Package constantfetch holds constantfetch related files -package constantfetch - -import ( - "bytes" - "errors" - "fmt" - "sort" - "text/template" - - "github.com/DataDog/datadog-go/v5/statsd" - - "github.com/DataDog/datadog-agent/pkg/ebpf" - "github.com/DataDog/datadog-agent/pkg/ebpf/bytecode/runtime" - "github.com/DataDog/datadog-agent/pkg/security/seclog" - "github.com/DataDog/datadog-agent/pkg/util/safeelf" -) - -type rcSymbolPair struct { - ID string - Operation string -} - -// RuntimeCompilationConstantFetcher is a constant fetcher utilizing runtime compilation -type RuntimeCompilationConstantFetcher struct { - config *ebpf.Config - statsdClient statsd.ClientInterface - headers []string - symbolPairs []rcSymbolPair - result map[string]uint64 -} - -// NewRuntimeCompilationConstantFetcher creates a RuntimeCompilationConstantFetcher -func NewRuntimeCompilationConstantFetcher(config *ebpf.Config, statsdClient statsd.ClientInterface) *RuntimeCompilationConstantFetcher { - return &RuntimeCompilationConstantFetcher{ - config: config, - statsdClient: statsdClient, - result: make(map[string]uint64), - } -} - -func (cf *RuntimeCompilationConstantFetcher) String() string { - return "runtime-compilation" -} - -// AppendSizeofRequest appends a sizeof request -func (cf *RuntimeCompilationConstantFetcher) AppendSizeofRequest(id, typeName, headerName string) { - cf.result[id] = ErrorSentinel - if headerName == "" { - return - } - - cf.headers = append(cf.headers, headerName) - cf.symbolPairs = append(cf.symbolPairs, rcSymbolPair{ - ID: id, - Operation: fmt.Sprintf("sizeof(%s)", typeName), - }) -} - -// AppendOffsetofRequest appends an offset request -func (cf *RuntimeCompilationConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName, headerName string) { - cf.result[id] = ErrorSentinel - if headerName == "" { - return - } - - cf.headers = append(cf.headers, headerName) - cf.symbolPairs = append(cf.symbolPairs, rcSymbolPair{ - ID: id, - Operation: fmt.Sprintf("offsetof(%s, %s)", typeName, fieldName), - }) -} - -const runtimeCompilationTemplate = ` -#include -#ifdef CONFIG_HAVE_ARCH_COMPILER_H -#include -#endif -{{ range .headers }} -#include <{{ . }}> -{{ end }} - -{{ range .symbols }} -size_t {{.ID}} = {{.Operation}}; -{{ end }} -` - -func (cf *RuntimeCompilationConstantFetcher) getCCode() (string, error) { - headers := sortAndDedup(cf.headers) - tmpl, err := template.New("runtimeCompilationTemplate").Parse(runtimeCompilationTemplate) - if err != nil { - return "", err - } - - var buffer bytes.Buffer - if err := tmpl.Execute(&buffer, map[string]interface{}{ - "headers": headers, - "symbols": cf.symbolPairs, - }); err != nil { - return "", err - } - - return buffer.String(), nil -} - -// FinishAndGetResults returns the results -func (cf *RuntimeCompilationConstantFetcher) FinishAndGetResults() (map[string]uint64, error) { - cCode, err := cf.getCCode() - if err != nil { - return nil, err - } - - elfFile, err := runtime.ConstantFetcher.Compile(cf.config, cCode, nil, cf.statsdClient) - if err != nil { - return nil, err - } - - f, err := safeelf.NewFile(elfFile) - if err != nil { - return nil, err - } - - symbols, err := f.Symbols() - if err != nil { - return nil, err - } - for _, sym := range symbols { - if _, present := cf.result[sym.Name]; !present { - continue - } - - section := f.Sections[sym.Section] - if section.ReaderAt == nil { - return nil, errors.New("section not available in random-access form") - } - buf := make([]byte, sym.Size) - if _, err := section.ReadAt(buf, int64(sym.Value)); err != nil { - return nil, fmt.Errorf("unable to read section at %d: %s", int64(sym.Value), err) - } - - var value uint64 - switch sym.Size { - case 4: - value = uint64(f.ByteOrder.Uint32(buf)) - case 8: - value = f.ByteOrder.Uint64(buf) - default: - return nil, fmt.Errorf("unexpected symbol size: `%v`", sym.Size) - } - - cf.result[sym.Name] = value - } - - seclog.Infof("runtime compiled constants: %v", cf.result) - return cf.result, nil -} - -func sortAndDedup(in []string) []string { - // sort and dedup headers - set := make(map[string]bool) - for _, value := range in { - set[value] = true - } - - out := make([]string, 0, len(in)) - for value := range set { - out = append(out, value) - } - sort.Strings(out) - return out -} diff --git a/pkg/security/probe/field_handlers_ebpf.go b/pkg/security/probe/field_handlers_ebpf.go index b76b70a76b833..63215156e4a19 100644 --- a/pkg/security/probe/field_handlers_ebpf.go +++ b/pkg/security/probe/field_handlers_ebpf.go @@ -20,6 +20,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/resolvers" sprocess "github.com/DataDog/datadog-agent/pkg/security/resolvers/process" "github.com/DataDog/datadog-agent/pkg/security/secl/containerutils" + "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/security/secl/args" "github.com/DataDog/datadog-agent/pkg/security/secl/model" @@ -230,7 +231,7 @@ func (fh *EBPFFieldHandlers) ResolveRights(_ *model.Event, e *model.FileFields) return int(e.Mode) & (syscall.S_ISUID | syscall.S_ISGID | syscall.S_ISVTX | syscall.S_IRWXU | syscall.S_IRWXG | syscall.S_IRWXO) } -// ResolveChownUID resolves the ResolveProcessCacheEntry id of a chown event to a username +// ResolveChownUID resolves the user id of a chown event to a username func (fh *EBPFFieldHandlers) ResolveChownUID(ev *model.Event, e *model.ChownEvent) string { if len(e.User) == 0 { e.User, _ = fh.resolvers.UserGroupResolver.ResolveUser(int(e.UID), ev.ContainerContext.ContainerID) @@ -731,3 +732,17 @@ func (fh *EBPFFieldHandlers) ResolveOnDemandArg4Str(_ *model.Event, e *model.OnD func (fh *EBPFFieldHandlers) ResolveOnDemandArg4Uint(_ *model.Event, e *model.OnDemandEvent) int { return int(binary.NativeEndian.Uint64(e.Data[192 : 192+8])) } + +// ResolveProcessNSID resolves the process namespace ID +func (fh *EBPFFieldHandlers) ResolveProcessNSID(e *model.Event) (uint64, error) { + if e.ProcessCacheEntry.Process.NSID != 0 { + return e.ProcessCacheEntry.Process.NSID, nil + } + + nsid, err := utils.GetProcessPidNamespace(e.ProcessCacheEntry.Process.Pid) + if err != nil { + return 0, err + } + e.ProcessCacheEntry.Process.NSID = nsid + return nsid, nil +} diff --git a/pkg/security/probe/probe_ebpf.go b/pkg/security/probe/probe_ebpf.go index f06cbe8fa45ae..828a8e0b85ce0 100644 --- a/pkg/security/probe/probe_ebpf.go +++ b/pkg/security/probe/probe_ebpf.go @@ -818,15 +818,11 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)); err != nil { + cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)) + if err != nil { seclog.Debugf("Failed to resolve cgroup: %s", err) } else { event.CgroupTracing.CGroupContext = *cgroupContext - if cgroupContext.CGroupFlags.IsContainer() { - containerID, _ := containerutils.FindContainerID(cgroupContext.CGroupID) - event.CgroupTracing.ContainerContext.ContainerID = containerID - } - p.profileManagers.activityDumpManager.HandleCGroupTracingEvent(&event.CgroupTracing) } @@ -886,14 +882,10 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if process.IsKThread(event.ProcessCacheEntry.PPid, event.ProcessCacheEntry.Pid) { + if err := p.Resolvers.ProcessResolver.AddForkEntry(event, newEntryCb); err != nil { + seclog.Errorf("failed to insert fork event: %s (pid %d, offset %d, len %d)", err, event.PIDContext.Pid, offset, len(data)) return } - - p.Resolvers.ProcessResolver.ApplyBootTime(event.ProcessCacheEntry) - event.ProcessCacheEntry.SetSpan(event.SpanContext.SpanID, event.SpanContext.TraceID) - - p.Resolvers.ProcessResolver.AddForkEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode, newEntryCb) case model.ExecEventType: // unmarshal and fill event.processCacheEntry if _, err = p.unmarshalProcessCacheEntry(event, data[offset:]); err != nil { @@ -901,18 +893,12 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if err = p.Resolvers.ProcessResolver.ResolveNewProcessCacheEntry(event.ProcessCacheEntry, event.ContainerContext); err != nil { - seclog.Debugf("failed to resolve new process cache entry context for pid %d: %s", event.PIDContext.Pid, err) - - var errResolution *path.ErrPathResolution - if errors.As(err, &errResolution) { - event.SetPathResolutionError(&event.ProcessCacheEntry.FileEvent, err) - } - } else { - p.Resolvers.ProcessResolver.AddExecEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode) + err = p.Resolvers.ProcessResolver.AddExecEntry(event) + if err != nil { + seclog.Errorf("failed to insert exec event: %s (pid %d, offset %d, len %d)", err, event.PIDContext.Pid, offset, len(data)) + return } - event.Exec.Process = &event.ProcessCacheEntry.Process } if !p.setProcessContext(eventType, event, newEntryCb) { @@ -1023,26 +1009,13 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { seclog.Errorf("failed to decode exit event: %s (offset %d, len %d)", err, offset, len(data)) return } - - var exists bool - event.ProcessCacheEntry, exists = p.fieldHandlers.GetProcessCacheEntry(event, newEntryCb) - if !exists { - // no need to dispatch an exit event that don't have the corresponding cache entry - return + exists := p.Resolvers.ProcessResolver.ApplyExitEntry(event, newEntryCb) + if exists { + p.Resolvers.MountResolver.DelPid(event.Exit.Pid) + // update action reports + p.processKiller.HandleProcessExited(event) + p.fileHasher.HandleProcessExited(event) } - - // Use the event timestamp as exit time - // The local process cache hasn't been updated yet with the exit time when the exit event is first seen - // The pid_cache kernel map has the exit_time but it's only accessed if there's a local miss - event.ProcessCacheEntry.Process.ExitTime = p.fieldHandlers.ResolveEventTime(event, &event.BaseEvent) - event.Exit.Process = &event.ProcessCacheEntry.Process - - // update mount pid mapping - p.Resolvers.MountResolver.DelPid(event.Exit.Pid) - - // update action reports - p.processKiller.HandleProcessExited(event) - p.fileHasher.HandleProcessExited(event) case model.SetuidEventType: // the process context may be incorrect, do not modify it if event.Error != nil { @@ -1117,7 +1090,14 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { if containerID == "" && event.PTrace.Request != unix.PTRACE_ATTACH { pidToResolve = event.PTrace.NSPID } else { - pid, err := utils.TryToResolveTraceePid(event.ProcessContext.Process.Pid, event.PTrace.NSPID) + nsid, err := p.fieldHandlers.ResolveProcessNSID(event) + if err != nil { + seclog.Debugf("PTrace NSID resolution error for process %s in container %s: %v", + event.ProcessContext.Process.FileEvent.PathnameStr, containerID, err) + return + } + + pid, err := utils.TryToResolveTraceePid(event.ProcessContext.Process.Pid, nsid, event.PTrace.NSPID) if err != nil { seclog.Debugf("PTrace tracee resolution error for process %s in container %s: %v", event.ProcessContext.Process.FileEvent.PathnameStr, containerID, err) @@ -2014,8 +1994,6 @@ func NewEBPFProbe(probe *Probe, config *config.Config, opts Opts) (*EBPFProbe, e p.supportsBPFSendSignal = p.kernelVersion.SupportBPFSendSignal() - p.ensureConfigDefaults() - p.monitors = NewEBPFMonitors(p) p.numCPU, err = utils.NumCPU() @@ -2272,13 +2250,6 @@ func (p *EBPFProbe) GetProfileManagers() *SecurityProfileManagers { return p.profileManagers } -func (p *EBPFProbe) ensureConfigDefaults() { - // enable runtime compiled constants on COS by default - if !p.config.Probe.RuntimeCompiledConstantsIsSet && p.kernelVersion.IsCOSKernel() { - p.config.Probe.RuntimeCompiledConstantsEnabled = true - } -} - const ( netStructHasProcINum uint64 = 0 netStructHasNS uint64 = 1 @@ -2407,123 +2378,135 @@ func getCGroupWriteConstants() manager.ConstantEditor { // GetOffsetConstants returns the offsets and struct sizes constants func (p *EBPFProbe) GetOffsetConstants() (map[string]uint64, error) { - constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion, p.statsdClient)) + constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion)) AppendProbeRequestsToFetcher(constantFetcher, p.kernelVersion) return constantFetcher.FinishAndGetResults() } // GetConstantFetcherStatus returns the status of the constant fetcher associated with this probe func (p *EBPFProbe) GetConstantFetcherStatus() (*constantfetch.ConstantFetcherStatus, error) { - constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion, p.statsdClient)) + constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion)) AppendProbeRequestsToFetcher(constantFetcher, p.kernelVersion) return constantFetcher.FinishAndGetStatus() } // AppendProbeRequestsToFetcher returns the offsets and struct sizes constants, from a constant fetcher func AppendProbeRequestsToFetcher(constantFetcher constantfetch.ConstantFetcher, kv *kernel.Version) { - constantFetcher.AppendSizeofRequest(constantfetch.SizeOfInode, "struct inode", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSFlags, "struct super_block", "s_flags", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSMagic, "struct super_block", "s_magic", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryStructDSB, "struct dentry", "d_sb", "linux/dcache.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSignalStructStructTTY, "struct signal_struct", "tty", "linux/sched/signal.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTTYStructStructName, "struct tty_struct", "name", "linux/tty.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructUID, "struct cred", "uid", "linux/cred.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructCapInheritable, "struct cred", "cap_inheritable", "linux/cred.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmP, "struct linux_binprm", "p", "linux/binfmts.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmArgc, "struct linux_binprm", "argc", "linux/binfmts.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmEnvc, "struct linux_binprm", "envc", "linux/binfmts.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameVMAreaStructFlags, "struct vm_area_struct", "vm_flags", "linux/mm_types.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFinode, "struct file", "f_inode", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFpath, "struct file", "f_path", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryDSb, "struct dentry", "d_sb", "linux/dcache.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameMountMntID, "struct mount", "mnt_id", "") + constantFetcher.AppendSizeofRequest(constantfetch.SizeOfInode, "struct inode") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSFlags, "struct super_block", "s_flags") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSMagic, "struct super_block", "s_magic") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryStructDSB, "struct dentry", "d_sb") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSignalStructStructTTY, "struct signal_struct", "tty") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTTYStructStructName, "struct tty_struct", "name") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructUID, "struct cred", "uid") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructCapInheritable, "struct cred", "cap_inheritable") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmP, "struct linux_binprm", "p") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmArgc, "struct linux_binprm", "argc") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmEnvc, "struct linux_binprm", "envc") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameVMAreaStructFlags, "struct vm_area_struct", "vm_flags") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFinode, "struct file", "f_inode") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFpath, "struct file", "f_path") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryDSb, "struct dentry", "d_sb") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameMountMntID, "struct mount", "mnt_id") if kv.Code >= kernel.Kernel5_3 { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameKernelCloneArgsExitSignal, "struct kernel_clone_args", "exit_signal", "linux/sched/task.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameKernelCloneArgsExitSignal, "struct kernel_clone_args", "exit_signal") } // inode time offsets // no runtime compilation for those, we expect them to default to 0 if not there and use the fallback // in the eBPF code itself in that case if kv.Code >= kernel.Kernel6_11 { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeSec, "struct inode", "i_ctime_sec", "") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeNsec, "struct inode", "i_ctime_nsec", "") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeSec, "struct inode", "i_mtime_sec", "") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeNsec, "struct inode", "i_mtime_nsec", "") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeSec, "struct inode", "i_ctime_sec") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeNsec, "struct inode", "i_ctime_nsec") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeSec, "struct inode", "i_mtime_sec") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeNsec, "struct inode", "i_mtime_nsec") } // rename offsets if kv.Code >= kernel.Kernel5_12 || (kv.IsInRangeCloseOpen(kernel.Kernel5_10, kernel.Kernel5_11) && kv.Code.Patch() >= 220) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructOldDentry, "struct renamedata", "old_dentry", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructNewDentry, "struct renamedata", "new_dentry", "linux/fs.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructOldDentry, "struct renamedata", "old_dentry") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructNewDentry, "struct renamedata", "new_dentry") } // bpf offsets - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructID, "struct bpf_map", "id", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructID, "struct bpf_map", "id") if kv.Code != 0 && (kv.Code >= kernel.Kernel4_15 || kv.IsRH7Kernel()) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructName, "struct bpf_map", "name", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructName, "struct bpf_map", "name") } - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructMapType, "struct bpf_map", "map_type", "linux/bpf.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructID, "struct bpf_prog_aux", "id", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructMapType, "struct bpf_map", "map_type") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructID, "struct bpf_prog_aux", "id") if kv.Code != 0 && (kv.Code >= kernel.Kernel4_15 || kv.IsRH7Kernel()) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructName, "struct bpf_prog_aux", "name", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructName, "struct bpf_prog_aux", "name") } - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructTag, "struct bpf_prog", "tag", "linux/filter.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructAux, "struct bpf_prog", "aux", "linux/filter.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructType, "struct bpf_prog", "type", "linux/filter.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructTag, "struct bpf_prog", "tag") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructAux, "struct bpf_prog", "aux") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructType, "struct bpf_prog", "type") if kv.Code != 0 && (kv.Code > kernel.Kernel4_16 || kv.IsSuse12Kernel() || kv.IsSuse15Kernel()) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructExpectedAttachType, "struct bpf_prog", "expected_attach_type", "linux/filter.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructExpectedAttachType, "struct bpf_prog", "expected_attach_type") } // namespace nr offsets - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructLevel, "struct pid", "level", "linux/pid.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructNumbers, "struct pid", "numbers", "linux/pid.h") - constantFetcher.AppendSizeofRequest(constantfetch.SizeOfUPID, "struct upid", "linux/pid.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructLevel, "struct pid", "level") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructNumbers, "struct pid", "numbers") + constantFetcher.AppendSizeofRequest(constantfetch.SizeOfUPID, "struct upid") if kv.HavePIDLinkStruct() { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPIDLink, "struct task_struct", "pids", "linux/sched.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDLinkStructPID, "struct pid_link", "pid", "linux/pid.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPIDLink, "struct task_struct", "pids") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDLinkStructPID, "struct pid_link", "pid") } else { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPID, "struct task_struct", "thread_pid", "linux/sched.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPID, "struct task_struct", "thread_pid") } // splice event - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBufs, "struct pipe_inode_info", "bufs", "linux/pipe_fs_i.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBufs, "struct pipe_inode_info", "bufs") if kv.HaveLegacyPipeInodeInfoStruct() { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructNrbufs, "struct pipe_inode_info", "nrbufs", "linux/pipe_fs_i.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructCurbuf, "struct pipe_inode_info", "curbuf", "linux/pipe_fs_i.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBuffers, "struct pipe_inode_info", "buffers", "linux/pipe_fs_i.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructNrbufs, "struct pipe_inode_info", "nrbufs") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructCurbuf, "struct pipe_inode_info", "curbuf") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBuffers, "struct pipe_inode_info", "buffers") } else { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructHead, "struct pipe_inode_info", "head", "linux/pipe_fs_i.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructRingsize, "struct pipe_inode_info", "ring_size", "linux/pipe_fs_i.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructHead, "struct pipe_inode_info", "head") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructRingsize, "struct pipe_inode_info", "ring_size") } // network related constants - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructIfIndex, "struct net_device", "ifindex", "linux/netdevice.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructName, "struct net_device", "name", "linux/netdevice.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDeviceStructNdNet, "struct net_device", "nd_net", "linux/netdevice.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCNet, "struct sock_common", "skc_net", "net/sock.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCFamily, "struct sock_common", "skc_family", "net/sock.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructSADDR, "struct flowi4", "saddr", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructULI, "struct flowi4", "uli", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructSADDR, "struct flowi6", "saddr", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructULI, "struct flowi6", "uli", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSocketStructSK, "struct socket", "sk", "linux/net.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructIfIndex, "struct net_device", "ifindex") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructName, "struct net_device", "name") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDeviceStructNdNet, "struct net_device", "nd_net") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCNet, "struct sock_common", "skc_net") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCFamily, "struct sock_common", "skc_family") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructSADDR, "struct flowi4", "saddr") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructULI, "struct flowi4", "uli") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructSADDR, "struct flowi6", "saddr") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructULI, "struct flowi6", "uli") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSocketStructSK, "struct socket", "sk") // Interpreter constants - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmStructFile, "struct linux_binprm", "file", "linux/binfmts.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmStructFile, "struct linux_binprm", "file") if !kv.IsRH7Kernel() { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNFConnStructCTNet, "struct nf_conn", "ct_net", "net/netfilter/nf_conntrack.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNFConnStructCTNet, "struct nf_conn", "ct_net") } if getNetStructType(kv) == netStructHasProcINum { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructProcInum, "struct net", "proc_inum", "net/net_namespace.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructProcInum, "struct net", "proc_inum") } else { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructNS, "struct net", "ns", "net/net_namespace.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructNS, "struct net", "ns") } // iouring if kv.Code != 0 && (kv.Code >= kernel.Kernel5_1) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameIoKiocbStructCtx, "struct io_kiocb", "ctx", "") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameIoKiocbStructCtx, "struct io_kiocb", "ctx") + } + + // inode + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeIno, "struct inode", "i_ino") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeGid, "struct inode", "i_gid") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeNlink, "struct inode", "i_nlink") + if kv.IsInRangeCloseOpen(kernel.Kernel6_7, kernel.Kernel6_11) { + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeMtime, "struct inode", "__i_mtime") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeCtime, "struct inode", "__i_ctime") + } else if kv.Code < kernel.Kernel6_7 { + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeMtime, "struct inode", "i_mtime") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeCtime, "struct inode", "i_ctime") } } diff --git a/pkg/security/resolvers/cgroup/resolver.go b/pkg/security/resolvers/cgroup/resolver.go index 2137cccf14506..dcb1de1e6d378 100644 --- a/pkg/security/resolvers/cgroup/resolver.go +++ b/pkg/security/resolvers/cgroup/resolver.go @@ -48,6 +48,7 @@ type ResolverInterface interface { type Resolver struct { *utils.Notifier[Event, *cgroupModel.CacheEntry] sync.Mutex + cgroups *simplelru.LRU[model.PathKey, *model.CGroupContext] hostWorkloads *simplelru.LRU[containerutils.CGroupID, *cgroupModel.CacheEntry] containerWorkloads *simplelru.LRU[containerutils.ContainerID, *cgroupModel.CacheEntry] } @@ -80,6 +81,11 @@ func NewResolver() (*Resolver, error) { return nil, err } + cr.cgroups, err = simplelru.NewLRU(2048, func(_ model.PathKey, _ *model.CGroupContext) {}) + if err != nil { + return nil, err + } + return cr, nil } @@ -121,10 +127,19 @@ func (cr *Resolver) AddPID(process *model.ProcessCacheEntry) { } else { cr.hostWorkloads.Add(process.CGroup.CGroupID, newCGroup) } + cr.cgroups.Add(process.CGroup.CGroupFile, &process.CGroup) cr.NotifyListeners(CGroupCreated, newCGroup) } +// GetCGroupContext returns the cgroup context with the specified path key +func (cr *Resolver) GetCGroupContext(cgroupPath model.PathKey) (*model.CGroupContext, bool) { + cr.Lock() + defer cr.Unlock() + + return cr.cgroups.Get(cgroupPath) +} + // GetWorkload returns the workload referenced by the provided ID func (cr *Resolver) GetWorkload(id containerutils.ContainerID) (*cgroupModel.CacheEntry, bool) { if id == "" { @@ -171,6 +186,7 @@ func (cr *Resolver) deleteWorkloadPID(pid uint32, workload *cgroupModel.CacheEnt // check if the workload should be deleted if len(workload.PIDs) <= 0 { + cr.cgroups.Remove(workload.CGroupFile) cr.hostWorkloads.Remove(workload.CGroupID) if workload.ContainerID != "" { cr.containerWorkloads.Remove(workload.ContainerID) @@ -183,5 +199,5 @@ func (cr *Resolver) Len() int { cr.Lock() defer cr.Unlock() - return cr.hostWorkloads.Len() + cr.containerWorkloads.Len() + return cr.cgroups.Len() } diff --git a/pkg/security/resolvers/process/resolver_ebpf.go b/pkg/security/resolvers/process/resolver_ebpf.go index dc5694dd55647..0e80c8aa2ae93 100644 --- a/pkg/security/resolvers/process/resolver_ebpf.go +++ b/pkg/security/resolvers/process/resolver_ebpf.go @@ -290,27 +290,61 @@ func (p *EBPFResolver) UpdateArgsEnvs(event *model.ArgsEnvsEvent) { } // AddForkEntry adds an entry to the local cache and returns the newly created entry -func (p *EBPFResolver) AddForkEntry(entry *model.ProcessCacheEntry, inode uint64, newEntryCb func(*model.ProcessCacheEntry, error)) { - if entry.Pid == 0 { - return +func (p *EBPFResolver) AddForkEntry(event *model.Event, newEntryCb func(*model.ProcessCacheEntry, error)) error { + p.ApplyBootTime(event.ProcessCacheEntry) + event.ProcessCacheEntry.SetSpan(event.SpanContext.SpanID, event.SpanContext.TraceID) + + if event.ProcessCacheEntry.Pid == 0 { + return errors.New("no pid") + } + if IsKThread(event.ProcessCacheEntry.PPid, event.ProcessCacheEntry.Pid) { + return errors.New("process is kthread") } p.Lock() defer p.Unlock() - - p.insertForkEntry(entry, inode, model.ProcessCacheEntryFromEvent, newEntryCb) + p.insertForkEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode, model.ProcessCacheEntryFromEvent, newEntryCb) + return nil } // AddExecEntry adds an entry to the local cache and returns the newly created entry -func (p *EBPFResolver) AddExecEntry(entry *model.ProcessCacheEntry, inode uint64) { - if entry.Pid == 0 { - return - } - +func (p *EBPFResolver) AddExecEntry(event *model.Event) error { p.Lock() defer p.Unlock() - p.insertExecEntry(entry, inode, model.ProcessCacheEntryFromEvent) + var err error + if err := p.ResolveNewProcessCacheEntry(event.ProcessCacheEntry, event.ContainerContext); err != nil { + var errResolution *spath.ErrPathResolution + if errors.As(err, &errResolution) { + event.SetPathResolutionError(&event.ProcessCacheEntry.FileEvent, err) + } + } else { + if event.ProcessCacheEntry.Pid == 0 { + return errors.New("no pid context") + } + p.insertExecEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode, model.ProcessCacheEntryFromEvent) + } + + event.Exec.Process = &event.ProcessCacheEntry.Process + + return err +} + +// ApplyExitEntry delete entry from the local cache if present +func (p *EBPFResolver) ApplyExitEntry(event *model.Event, newEntryCb func(*model.ProcessCacheEntry, error)) bool { + event.ProcessCacheEntry = p.resolve(event.PIDContext.Pid, event.PIDContext.Tid, event.PIDContext.ExecInode, false, newEntryCb) + if event.ProcessCacheEntry == nil { + // no need to dispatch an exit event that don't have the corresponding cache entry + return false + } + + // Use the event timestamp as exit time + // The local process cache hasn't been updated yet with the exit time when the exit event is first seen + // The pid_cache kernel map has the exit_time but it's only accessed if there's a local miss + event.ProcessCacheEntry.ExitTime = event.FieldHandlers.ResolveEventTime(event, &event.BaseEvent) + event.Exit.Process = &event.ProcessCacheEntry.Process + return true + } // enrichEventFromProc uses /proc to enrich a ProcessCacheEntry with additional metadata @@ -334,13 +368,16 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc // Get the file fields of the process binary info, err := p.retrieveExecFileFields(procExecPath) if err != nil { + if !os.IsNotExist(err) { + seclog.Errorf("snapshot failed for %d: couldn't retrieve inode info: %s", proc.Pid, err) + } return fmt.Errorf("snapshot failed for %d: couldn't retrieve inode info: %w", proc.Pid, err) } // Retrieve the container ID of the process from /proc containerID, cgroup, err := p.containerResolver.GetContainerContext(pid) if err != nil { - return fmt.Errorf("snapshot failed for %d: couldn't parse container ID: %w", proc.Pid, err) + return fmt.Errorf("snapshot failed for %d: couldn't parse container and cgroup context: %w", proc.Pid, err) } entry.ContainerID = containerID @@ -465,11 +502,11 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc func (p *EBPFResolver) retrieveExecFileFields(procExecPath string) (*model.FileFields, error) { fi, err := os.Stat(procExecPath) if err != nil { - return nil, fmt.Errorf("snapshot failed for `%s`: couldn't stat binary: %w", procExecPath, err) + return nil, err } stat, ok := fi.Sys().(*syscall.Stat_t) if !ok { - return nil, fmt.Errorf("snapshot failed for `%s`: couldn't stat binary", procExecPath) + return nil, errors.New("wrong type") } inode := stat.Ino @@ -483,11 +520,11 @@ func (p *EBPFResolver) retrieveExecFileFields(procExecPath string) (*model.FileF var fileFields model.FileFields if _, err := fileFields.UnmarshalBinary(data); err != nil { - return nil, fmt.Errorf("unable to unmarshal entry for inode `%d`", inode) + return nil, fmt.Errorf("unable to unmarshal entry for inode `%d`: %v", inode, err) } if fileFields.Inode == 0 { - return nil, errors.New("not found") + return nil, fmt.Errorf("inode `%d` not found: %v", inode, err) } return &fileFields, nil @@ -520,6 +557,7 @@ func (p *EBPFResolver) insertEntry(entry, prev *model.ProcessCacheEntry, source } func (p *EBPFResolver) insertForkEntry(entry *model.ProcessCacheEntry, inode uint64, source uint64, newEntryCb func(*model.ProcessCacheEntry, error)) { + if entry.Pid == 0 { return } @@ -529,7 +567,6 @@ func (p *EBPFResolver) insertForkEntry(entry *model.ProcessCacheEntry, inode uin // this shouldn't happen but it is better to exit the prev and let the new one replace it prev.Exit(entry.ForkTime) } - if entry.Pid != 1 { parent := p.entryCache[entry.PPid] if entry.PPid >= 1 && inode != 0 && (parent == nil || parent.FileEvent.Inode != inode) { @@ -568,7 +605,6 @@ func (p *EBPFResolver) insertExecEntry(entry *model.ProcessCacheEntry, inode uin prev.ApplyExecTimeOf(entry) return } - prev.Exec(entry) } else { entry.IsParentMissing = true @@ -654,7 +690,6 @@ func (p *EBPFResolver) resolveFileFieldsPath(e *model.FileFields, pce *model.Pro err error maxDepthRetry = 3 ) - for maxDepthRetry > 0 { pathnameStr, mountPath, source, origin, err = p.pathResolver.ResolveFileFieldsPath(e, &pce.PIDContext, ctrCtx) if err == nil { @@ -686,7 +721,6 @@ func (p *EBPFResolver) SetProcessPath(fileEvent *model.FileEvent, pce *model.Pro if fileEvent.Inode == 0 { return onError("", &model.ErrInvalidKeyPath{Inode: fileEvent.Inode, MountID: fileEvent.MountID}) } - pathnameStr, mountPath, source, origin, err := p.resolveFileFieldsPath(&fileEvent.FileFields, pce, ctrCtx) if err != nil { return onError(pathnameStr, err) @@ -852,7 +886,7 @@ func (p *EBPFResolver) resolveFromKernelMaps(pid, tid uint32, inode uint64, newE // is no insurance that the parent of this process is still running, we can't use our user space cache to check if // the parent is in a container. In other words, we have to fall back to /proc to query the container ID of the // process. - if entry.ContainerID == "" || entry.CGroup.CGroupFile.Inode == 0 { + if entry.CGroup.CGroupFile.Inode == 0 { if containerID, cgroup, err := p.containerResolver.GetContainerContext(pid); err == nil { entry.CGroup.Merge(&cgroup) entry.ContainerID = containerID diff --git a/pkg/security/resolvers/process/resolver_test.go b/pkg/security/resolvers/process/resolver_test.go index c2febee639beb..16432e67338e1 100644 --- a/pkg/security/resolvers/process/resolver_test.go +++ b/pkg/security/resolvers/process/resolver_test.go @@ -16,10 +16,44 @@ import ( "github.com/avast/retry-go/v4" "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/pkg/security/resolvers/cgroup" + "github.com/DataDog/datadog-agent/pkg/security/resolvers/path" + "github.com/DataDog/datadog-agent/pkg/security/resolvers/usergroup" "github.com/DataDog/datadog-agent/pkg/security/secl/model" + "github.com/DataDog/datadog-agent/pkg/util/ktime" "github.com/DataDog/datadog-go/v5/statsd" ) +func newFakeForkEvent(ppid, pid int, inode uint64, resolver *EBPFResolver) *model.Event { + e := model.NewFakeEvent() + e.Type = uint32(model.ForkEventType) + e.ProcessCacheEntry = resolver.NewProcessCacheEntry(model.PIDContext{Pid: uint32(pid), Tid: uint32(pid)}) + e.PIDContext = e.ProcessCacheEntry.PIDContext + e.ProcessContext = &e.ProcessCacheEntry.ProcessContext + e.ProcessCacheEntry.ForkTime = time.Now() + e.ProcessCacheEntry.PPid = uint32(ppid) + e.ProcessCacheEntry.Pid = uint32(pid) + e.ProcessCacheEntry.FileEvent.Inode = inode + return e +} + +func newFakeExecEvent(ppid, pid int, inode uint64, resolver *EBPFResolver) *model.Event { + e := model.NewFakeEvent() + e.Type = uint32(model.ExecEventType) + e.ProcessCacheEntry = resolver.NewProcessCacheEntry(model.PIDContext{Pid: uint32(pid), Tid: uint32(pid)}) + e.PIDContext = e.ProcessCacheEntry.PIDContext + e.ProcessContext = &e.ProcessCacheEntry.ProcessContext + e.ProcessCacheEntry.ExecTime = time.Now() + e.ProcessCacheEntry.PPid = uint32(ppid) + e.ProcessCacheEntry.Pid = uint32(pid) + e.ProcessCacheEntry.FileEvent.Inode = inode + return e +} + +func exit(event *model.Event) { + event.Type = uint32(model.ExitEventType) +} + func testCacheSize(t *testing.T, resolver *EBPFResolver) { err := retry.Do( func() error { @@ -33,701 +67,717 @@ func testCacheSize(t *testing.T, resolver *EBPFResolver) { assert.NoError(t, err) } -func TestFork1st(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) +func newResolver() (*EBPFResolver, error) { + timeResolver, err := ktime.NewResolver() + if err != nil { + return nil, err + } + + cgroupsResolver, err := cgroup.NewResolver() + if err != nil { + return nil, err + } + + userGroupResolver, err := usergroup.NewResolver(cgroupsResolver) + if err != nil { + return nil, err + } + + resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, cgroupsResolver, userGroupResolver, timeResolver, &path.NoOpResolver{}, nil, NewResolverOpts()) if err != nil { - t.Fatal(err) + return nil, err } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() + return resolver, nil +} + +func TestFork1st(t *testing.T) { + + resolver, err := newResolver() + if err != nil { + t.Fatal() + } - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // parent - resolver.DeleteEntry(child.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[child.Pid]) - assert.Equal(t, 1, len(resolver.entryCache)) + // X(pid:3) + exit(child) + resolver.ApplyExitEntry(child, nil) + resolver.DeleteEntry(child.ProcessCacheEntry.Pid, child.ResolveEventTime()) - // nothing - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Zero(t, len(resolver.entryCache)) + assert.Nil(t, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) + // nothing in the entryCache + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessCacheEntry.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 0, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestFork2nd(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // nothing - resolver.DeleteEntry(child.Pid, time.Now()) - assert.Zero(t, len(resolver.entryCache)) + // nothing in the entryCache + exit(child) + resolver.ApplyExitEntry(child, nil) + resolver.DeleteEntry(child.ProcessContext.Pid, child.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 0, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec := newFakeExecEvent(3, 4, 456, resolver) - exec := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec.PPid = child.PPid - exec.FileEvent.Inode = 123 - exec.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // parent - // \ [child] -> exec - resolver.AddExecEntry(exec, 0) - assert.Equal(t, exec, resolver.entryCache[exec.Pid]) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec) + assert.Equal(t, exec.ProcessCacheEntry, resolver.entryCache[exec.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, child, exec.Ancestor) - assert.Equal(t, parent, exec.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent] - // \ [child] -> exec - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec.Ancestor) - assert.Equal(t, parent, exec.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor.Ancestor) - // nothing - resolver.DeleteEntry(exec.Pid, time.Now()) + // nothing in the entryCache + exit(child) + resolver.ApplyExitEntry(child, nil) + resolver.DeleteEntry(child.ProcessContext.Pid, child.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) - testCacheSize(t, resolver) } func TestOrphanExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec := newFakeExecEvent(3, 4, 456, resolver) - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() - - exec := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec.Pid = child.Pid - exec.PPid = child.PPid - exec.FileEvent.Inode = 123 - exec.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // [parent] - // \ [child] -> exec - resolver.AddExecEntry(exec, 0) - assert.Equal(t, exec, resolver.entryCache[exec.Pid]) + // [X(pid:3)] + // | + // X(pid:4) --> Y(pid:4) + resolver.AddExecEntry(exec) + assert.Equal(t, exec.ProcessCacheEntry, resolver.entryCache[exec.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec.Ancestor) - assert.Equal(t, parent, exec.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // nothing - resolver.DeleteEntry(exec.Pid, time.Now()) + // nothing in the entryCache + exit(exec) + resolver.ApplyExitEntry(exec, nil) + resolver.DeleteEntry(exec.ProcessCacheEntry.Pid, time.Now()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkExecExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() - - exec1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec1.PPid = child.PPid - exec1.FileEvent.Inode = 123 - exec1.ExecTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec1 := newFakeExecEvent(3, 4, 456, resolver) + exec2 := newFakeExecEvent(3, 4, 789, resolver) - exec2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec2.Pid = child.Pid - exec2.PPid = child.PPid - exec2.FileEvent.Inode = 456 - exec2.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // [parent] - // \ [child] -> exec1 - resolver.AddExecEntry(exec1, 0) - assert.Equal(t, exec1, resolver.entryCache[exec1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec1) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec1.Ancestor) - assert.Equal(t, parent, exec1.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent] - // \ [child] -> [exec1] -> exec2 - resolver.AddExecEntry(exec2, 0) - assert.Equal(t, exec2, resolver.entryCache[exec2.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) + resolver.AddExecEntry(exec2) + assert.Equal(t, exec2.ProcessCacheEntry, resolver.entryCache[exec2.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, exec1, exec2.Ancestor) - assert.Equal(t, child, exec2.Ancestor.Ancestor) - assert.Equal(t, parent, exec2.Ancestor.Ancestor.Ancestor) + assert.Equal(t, exec1.ProcessCacheEntry, exec2.ProcessCacheEntry.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec2.ProcessCacheEntry.Ancestor.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec2.ProcessCacheEntry.Ancestor.Ancestor.Ancestor) assert.EqualValues(t, 4, resolver.cacheSize.Load()) - // nothing - resolver.DeleteEntry(exec2.Pid, time.Now()) + // nothing in the entryCache in the entryCache + exit(exec2) + resolver.ApplyExitEntry(exec2, nil) + resolver.DeleteEntry(exec1.ProcessCacheEntry.Pid, time.Now()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkReuse(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent1.ForkTime = time.Now() - - child1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child1.PPid = parent1.Pid - child1.ForkTime = time.Now() - - exec1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child1.Pid, Tid: child1.Pid}) - exec1.PPid = child1.PPid - exec1.FileEvent.Inode = 123 - exec1.ExecTime = time.Now() + parent1 := newFakeForkEvent(0, 3, 123, resolver) + child1 := newFakeForkEvent(3, 4, 123, resolver) + exec1 := newFakeExecEvent(3, 4, 456, resolver) + parent2 := newFakeForkEvent(0, 3, 123, resolver) + child2 := newFakeForkEvent(3, 5, 123, resolver) - parent2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent2.ForkTime = time.Now() - - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 3, Tid: 3}) - child2.PPid = parent2.Pid - child2.ForkTime = time.Now() - - // parent1 - resolver.AddForkEntry(parent1, 0, nil) - assert.Equal(t, parent1, resolver.entryCache[parent1.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent1, nil) + assert.Equal(t, parent1.ProcessCacheEntry, resolver.entryCache[parent1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent1 - // \ child1 - resolver.AddForkEntry(child1, 0, nil) - assert.Equal(t, child1, resolver.entryCache[child1.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child1, nil) + assert.Equal(t, child1.ProcessCacheEntry, resolver.entryCache[child1.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent1, child1.Ancestor) + assert.Equal(t, parent1.ProcessCacheEntry, child1.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent1] - // \ child1 - resolver.DeleteEntry(parent1.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent1) + resolver.ApplyExitEntry(parent1, nil) + resolver.DeleteEntry(parent1.ProcessContext.Pid, parent1.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent1, child1.Ancestor) + assert.Equal(t, parent1.ProcessCacheEntry, child1.ProcessCacheEntry.Ancestor) - // [parent1] - // \ [child1] -> exec1 - resolver.AddExecEntry(exec1, 0) - assert.Equal(t, exec1, resolver.entryCache[exec1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec1) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child1, exec1.Ancestor) - assert.Equal(t, parent1, exec1.Ancestor.Ancestor) + assert.Equal(t, child1.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent1.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent1:pid1] - // \ [child1] -> exec1 + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) // - // parent2:pid1 - resolver.AddForkEntry(parent2, 0, nil) - assert.Equal(t, parent2, resolver.entryCache[parent2.Pid]) + // Z(pid:3) + resolver.AddForkEntry(parent2, nil) + assert.Equal(t, parent2.ProcessCacheEntry, resolver.entryCache[parent2.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) assert.EqualValues(t, 4, resolver.cacheSize.Load()) - // [parent1:pid1] - // \ [child1] -> exec1 + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) // - // parent2:pid1 - // \ child2 - resolver.AddForkEntry(child2, 0, nil) - assert.Equal(t, child2, resolver.entryCache[child2.Pid]) + // Z(pid:3) + // | + // T(pid:5) + resolver.AddForkEntry(child2, nil) + assert.Equal(t, child2.ProcessCacheEntry, resolver.entryCache[child2.ProcessCacheEntry.Pid]) assert.Equal(t, 3, len(resolver.entryCache)) - assert.Equal(t, parent2, child2.Ancestor) + assert.Equal(t, parent2.ProcessCacheEntry, child2.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 5, resolver.cacheSize.Load()) - // parent2:pid1 - // \ child2 - resolver.DeleteEntry(exec1.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[exec1.Pid]) + // Z(pid:3) + // | + // T(pid:5) + exit(exec1) + resolver.ApplyExitEntry(exec1, nil) + resolver.DeleteEntry(exec1.ProcessContext.Pid, exec1.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - // [parent2:pid1] - // \ child2 - resolver.DeleteEntry(parent2.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent2.Pid]) + // [Z(pid:3)] + // | + // T(pid:5) + exit(parent2) + resolver.ApplyExitEntry(parent2, nil) + resolver.DeleteEntry(parent2.ProcessContext.Pid, parent2.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent2.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent2, child2.Ancestor) + assert.Equal(t, parent2.ProcessCacheEntry, child2.ProcessCacheEntry.Ancestor) - // nothing - resolver.DeleteEntry(child2.Pid, time.Now()) + // nothing in the entryCache + exit(child2) + resolver.ApplyExitEntry(child2, nil) + resolver.DeleteEntry(child2.ProcessCacheEntry.Pid, child2.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkForkExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + grandChild := newFakeForkEvent(4, 5, 123, resolver) + childExec := newFakeExecEvent(3, 4, 456, resolver) - grandChild := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 3, Tid: 3}) - grandChild.PPid = child.Pid - grandChild.ForkTime = time.Now() - - childExec := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - childExec.Pid = child.Pid - childExec.PPid = child.PPid - childExec.FileEvent.Inode = 123 - childExec.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) - - // parent - // \ child - // \ grandChild - resolver.AddForkEntry(grandChild, 0, nil) - assert.Equal(t, grandChild, resolver.entryCache[grandChild.Pid]) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) + + // X(pid:3) + // | + // X(pid:4) + // | + // X(pid:5) + resolver.AddForkEntry(grandChild, nil) + assert.Equal(t, grandChild.ProcessCacheEntry, resolver.entryCache[grandChild.ProcessCacheEntry.Pid]) assert.Equal(t, 3, len(resolver.entryCache)) - assert.Equal(t, child, grandChild.Ancestor) - assert.Equal(t, parent, grandChild.Ancestor.Ancestor) - - // parent - // \ [child] -> childExec - // \ grandChild - resolver.AddExecEntry(childExec, 0) - assert.Equal(t, childExec, resolver.entryCache[childExec.Pid]) + assert.Equal(t, child.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor.Ancestor) + + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + // | + // X(pid:5) + resolver.AddExecEntry(childExec) + assert.Equal(t, childExec.ProcessCacheEntry, resolver.entryCache[childExec.ProcessCacheEntry.Pid]) assert.Equal(t, 3, len(resolver.entryCache)) - assert.Equal(t, child, childExec.Ancestor) - assert.Equal(t, parent, childExec.Ancestor.Ancestor) - assert.Equal(t, child, grandChild.Ancestor) - assert.Equal(t, parent, grandChild.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, childExec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, childExec.ProcessCacheEntry.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor.Ancestor) // [parent] // \ [child] -> childExec // \ grandChild - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + // | + // X(pid:5) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - // [parent] - // \ [child] - // \ grandChild - resolver.DeleteEntry(childExec.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[childExec.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + // | + // X(pid:5) + exit(childExec) + resolver.ApplyExitEntry(childExec, nil) + resolver.DeleteEntry(childExec.ProcessContext.Pid, childExec.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[childExec.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - // nothing - resolver.DeleteEntry(grandChild.Pid, time.Now()) + // nothing in the entryCache + exit(grandChild) + resolver.ApplyExitEntry(grandChild, nil) + resolver.DeleteEntry(grandChild.ProcessContext.Pid, grandChild.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestExecBomb(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec1 := newFakeExecEvent(3, 4, 456, resolver) + exec2 := newFakeExecEvent(3, 4, 456, resolver) - exec1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec1.PPid = child.PPid - exec1.FileEvent.Inode = 123 - exec1.ExecTime = time.Now() - - exec2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec2.Pid = child.Pid - exec2.PPid = child.PPid - exec2.FileEvent.Inode = 123 - exec2.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // [parent] - // \ [child] -> exec1 - resolver.AddExecEntry(exec1, 0) - assert.Equal(t, exec1, resolver.entryCache[exec1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec1) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec1.Ancestor) - assert.Equal(t, parent, exec1.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent] - // \ [child] -> [exec1] -> exec2 - resolver.AddExecEntry(exec2, 0) - assert.Equal(t, exec1, resolver.entryCache[exec2.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec2) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec2.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, exec1.ExecTime, exec2.ExecTime) + assert.Equal(t, exec1.ProcessCacheEntry.ExecTime, exec2.ProcessCacheEntry.ExecTime) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // nothing - resolver.DeleteEntry(exec1.Pid, time.Now()) + // nothing in the entryCache + exit(exec1) + resolver.ApplyExitEntry(exec1, nil) + resolver.DeleteEntry(exec1.ProcessContext.Pid, exec1.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestExecLostFork(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 11, Tid: 11}) - parent.FileEvent.BasenameStr = "agent" - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - parent.ExecInode = 1 - - // parent - resolver.AddForkEntry(parent, 0, nil) - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 22, Tid: 22}) - child.PPid = parent.Pid - child.FileEvent.Inode = 1 - - // parent - // \ child - resolver.AddForkEntry(child, parent.ExecInode, nil) - - assert.Equal(t, "agent", child.FileEvent.BasenameStr) - assert.False(t, child.IsParentMissing) - - // exec loss with inode 2 - - child1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 33, Tid: 33}) - child1.FileEvent.BasenameStr = "sh" - child1.PPid = child.Pid - child1.ExecInode = 2 - - // parent - // \ child - // \ child1 - resolver.AddForkEntry(child1, child1.ExecInode, nil) + parent := newFakeForkEvent(0, 3, 123, resolver) + parent.ProcessCacheEntry.FileEvent.BasenameStr = "agent" + child := newFakeForkEvent(3, 4, 123, resolver) + child.PIDContext.ExecInode = 123 // ExecInode == Inode Parent + child1 := newFakeForkEvent(4, 5, 123, resolver) + child1.ProcessCacheEntry.FileEvent.BasenameStr = "sh" + child1.PIDContext.ExecInode = 456 // ExecInode != Inode parent + + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, "agent", child1.FileEvent.BasenameStr) - assert.True(t, child1.IsParentMissing) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) + assert.Equal(t, "agent", child.ProcessCacheEntry.FileEvent.BasenameStr) + assert.False(t, child.ProcessCacheEntry.IsParentMissing) + + // X(pid:3) + // | + // X(pid:4) + // {|} + // X(pid:5) + resolver.AddForkEntry(child1, nil) + assert.Equal(t, "agent", child1.ProcessCacheEntry.FileEvent.BasenameStr) + assert.True(t, child1.ProcessCacheEntry.IsParentMissing) } func TestExecLostExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 11, Tid: 11}) - parent.FileEvent.BasenameStr = "agent" - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - parent.ExecInode = 1 - - // parent - resolver.AddForkEntry(parent, 0, nil) - - child1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 22, Tid: 22}) - child1.PPid = parent.Pid - child1.FileEvent.Inode = 1 - child1.ExecInode = 1 - - // parent - // \ child1 - resolver.AddForkEntry(child1, parent.ExecInode, nil) - - assert.Equal(t, "agent", child1.FileEvent.BasenameStr) - assert.False(t, child1.IsParentMissing) - - // exec loss with inode 2 and pid 22 - - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 33, Tid: 33}) - child2.FileEvent.BasenameStr = "sh" - child2.PPid = child1.Pid - child2.ExecInode = 2 - - // parent - // \ child1 - // \ child2 - resolver.AddForkEntry(child2, child2.ExecInode, nil) + parent := newFakeForkEvent(0, 3, 123, resolver) + parent.ProcessCacheEntry.FileEvent.BasenameStr = "agent" + child1 := newFakeForkEvent(3, 4, 123, resolver) + child1.PIDContext.ExecInode = 123 // ExecInode == Inode Parent + child2 := newFakeExecEvent(3, 4, 456, resolver) + child2.ProcessCacheEntry.FileEvent.BasenameStr = "sh" + child2.PIDContext.ExecInode = 456 // ExecInode != Inode Ancestor + + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, "agent", child2.FileEvent.BasenameStr) - assert.True(t, child2.IsParentMissing) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child1, nil) + assert.Equal(t, child1.ProcessCacheEntry, resolver.entryCache[child1.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child1.ProcessCacheEntry.Ancestor) + assert.Equal(t, "agent", child1.ProcessCacheEntry.FileEvent.BasenameStr) + assert.False(t, child1.ProcessCacheEntry.IsParentMissing) + + // X(pid:3) + // | + // X(pid:4) -**- Y(pid:4) + resolver.AddExecEntry(child2) + assert.NotEqual(t, "agent", child2.ProcessCacheEntry.FileEvent.BasenameStr) + assert.True(t, child2.ProcessCacheEntry.IsParentMissing) } func TestIsExecExecRuntime(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - - // parent - resolver.AddForkEntry(parent, 0, nil) + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + child2 := newFakeExecEvent(3, 4, 456, resolver) + child3 := newFakeExecEvent(3, 4, 789, resolver) + child4 := newFakeExecEvent(3, 4, 101112, resolver) - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.FileEvent.Inode = 1 - - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - // parent - // \ child - // \ child2 + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child2.FileEvent.Inode = 2 - child2.PPid = child.Pid - resolver.AddExecEntry(child2, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(child2) - // parent - // \ child - // \ child2 - // \ child3 + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) + resolver.AddExecEntry(child3) - child3 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child3.FileEvent.Inode = 3 - child3.PPid = child2.Pid - resolver.AddExecEntry(child3, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) -- T(pid:4) + resolver.AddExecEntry(child4) - assert.False(t, parent.IsExecExec) - assert.False(t, parent.IsExec) + assert.False(t, parent.ProcessCacheEntry.IsExecExec) + assert.False(t, parent.ProcessCacheEntry.IsExec) - assert.False(t, child.IsExecExec) - assert.False(t, child.IsExec) + assert.False(t, child.ProcessCacheEntry.IsExecExec) + assert.False(t, child.ProcessCacheEntry.IsExec) - assert.False(t, child2.IsExecExec) - assert.True(t, child2.IsExec) + assert.False(t, child2.ProcessCacheEntry.IsExecExec) + assert.True(t, child2.ProcessCacheEntry.IsExec) - assert.True(t, child3.IsExecExec) - assert.True(t, child3.IsExec) + assert.True(t, child3.ProcessCacheEntry.IsExecExec) + assert.True(t, child3.ProcessCacheEntry.IsExec) - child4 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child4.FileEvent.Inode = 3 - child4.PPid = child3.Pid - resolver.AddExecEntry(child4, 0) + assert.True(t, child4.ProcessCacheEntry.IsExecExec) + assert.True(t, child4.ProcessCacheEntry.IsExec) - assert.True(t, child3.IsExecExec) - assert.True(t, child3.IsExec) } func TestIsExecExecSnapshot(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - - // parent - resolver.insertEntry(parent, nil, model.ProcessCacheEntryFromSnapshot) - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.FileEvent.Inode = 2 + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + child2 := newFakeExecEvent(3, 4, 456, resolver) + child3 := newFakeExecEvent(3, 4, 769, resolver) - // parent - // \ child - - resolver.setAncestor(child) - resolver.insertEntry(child, nil, model.ProcessCacheEntryFromSnapshot) + // X(pid:3) + resolver.insertEntry(parent.ProcessCacheEntry, nil, model.ProcessCacheEntryFromSnapshot) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - assert.False(t, parent.IsExecExec) - assert.False(t, parent.IsExec) + // X(pid:3) + // | + // X(pid:4) + resolver.setAncestor(child.ProcessCacheEntry) + resolver.insertEntry(child.ProcessCacheEntry, nil, model.ProcessCacheEntryFromSnapshot) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - assert.False(t, child.IsExecExec) - assert.False(t, child.IsExec) + assert.False(t, parent.ProcessCacheEntry.IsExecExec) + assert.False(t, parent.ProcessCacheEntry.IsExec) - // parent - // \ child - // \ child2 + assert.False(t, child.ProcessCacheEntry.IsExecExec) + assert.False(t, child.ProcessCacheEntry.IsExec) - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child2.FileEvent.Inode = 3 - child2.PPid = child.Pid - resolver.AddExecEntry(child2, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(child2) - assert.False(t, child2.IsExecExec) - assert.True(t, child2.IsExec) + assert.False(t, child2.ProcessCacheEntry.IsExecExec) + assert.True(t, child2.ProcessCacheEntry.IsExec) - child3 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child3.FileEvent.Inode = 4 - child3.PPid = child2.Pid - resolver.AddExecEntry(child3, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) + resolver.AddExecEntry(child3) - assert.True(t, child3.IsExecExec) - assert.True(t, child3.IsExec) + assert.True(t, child3.ProcessCacheEntry.IsExecExec) + assert.True(t, child3.ProcessCacheEntry.IsExec) } diff --git a/pkg/security/resolvers/resolvers_ebpf.go b/pkg/security/resolvers/resolvers_ebpf.go index b8899bbd99adf..c9adeb82e557b 100644 --- a/pkg/security/resolvers/resolvers_ebpf.go +++ b/pkg/security/resolvers/resolvers_ebpf.go @@ -219,6 +219,10 @@ func (r *EBPFResolvers) Start(ctx context.Context) error { // ResolveCGroupContext resolves the cgroup context from a cgroup path key func (r *EBPFResolvers) ResolveCGroupContext(pathKey model.PathKey, cgroupFlags containerutils.CGroupFlags) (*model.CGroupContext, error) { + if cgroupContext, found := r.CGroupResolver.GetCGroupContext(pathKey); found { + return cgroupContext, nil + } + path, err := r.DentryResolver.Resolve(pathKey, true) if err != nil { return nil, fmt.Errorf("failed to resolve cgroup file %v: %w", pathKey, err) diff --git a/pkg/security/secl/compiler/eval/eval_test.go b/pkg/security/secl/compiler/eval/eval_test.go index 0674e4ef744b1..6a217e8877444 100644 --- a/pkg/security/secl/compiler/eval/eval_test.go +++ b/pkg/security/secl/compiler/eval/eval_test.go @@ -1006,8 +1006,9 @@ func TestRegisterPartial(t *testing.T) { func TestOptimizer(t *testing.T) { event := &testEvent{ process: testProcess{ - uid: 44, - gid: 44, + uid: 44, + gid: 44, + name: "aaa", }, } @@ -1018,10 +1019,11 @@ func TestOptimizer(t *testing.T) { Expr string Evaluated func() bool }{ - {Expr: `process.list[A].key == 44 && process.gid == 55`, Evaluated: func() bool { return event.listEvaluated }}, + {Expr: `process.list.key == 44 && process.gid == 55`, Evaluated: func() bool { return event.listEvaluated }}, {Expr: `process.gid == 55 && process.list[A].key == 44`, Evaluated: func() bool { return event.listEvaluated }}, {Expr: `process.uid in [66, 77, 88] && process.gid == 55`, Evaluated: func() bool { return event.uidEvaluated }}, {Expr: `process.gid == 55 && process.uid in [66, 77, 88]`, Evaluated: func() bool { return event.uidEvaluated }}, + {Expr: `process.list.value == "AA" && process.name == "zzz"`, Evaluated: func() bool { return event.listEvaluated }}, } for _, test := range tests { diff --git a/pkg/security/secl/compiler/eval/operators.go b/pkg/security/secl/compiler/eval/operators.go index 9f39ff3e3550e..9fd97b9a98177 100644 --- a/pkg/security/secl/compiler/eval/operators.go +++ b/pkg/security/secl/compiler/eval/operators.go @@ -352,7 +352,6 @@ func StringEquals(a *StringEvaluator, b *StringEvaluator, state *State) (*BoolEv return &BoolEvaluator{ Value: op(ea, eb), - Weight: a.Weight + InArrayWeight*len(eb), isDeterministic: isDc, }, nil } @@ -366,7 +365,7 @@ func StringEquals(a *StringEvaluator, b *StringEvaluator, state *State) (*BoolEv return &BoolEvaluator{ EvalFnc: evalFnc, - Weight: a.Weight + InArrayWeight*len(eb), + Weight: a.Weight, isDeterministic: isDc, }, nil } diff --git a/pkg/security/secl/containerutils/cgroup.go b/pkg/security/secl/containerutils/cgroup.go index 74e7a64540c84..9cc6cca12d884 100644 --- a/pkg/security/secl/containerutils/cgroup.go +++ b/pkg/security/secl/containerutils/cgroup.go @@ -24,6 +24,12 @@ const ( CGroupManagerSystemd // systemd ) +// CGroup flags +const ( + SystemdService CGroupFlags = (0 << 8) + SystemdScope CGroupFlags = (1 << 8) +) + const ( // ContainerRuntimeDocker is used to specify that a container is managed by Docker ContainerRuntimeDocker = "docker" diff --git a/pkg/security/secl/containerutils/helpers.go b/pkg/security/secl/containerutils/helpers.go index 0e46e87af7f2a..311322f94ffc6 100644 --- a/pkg/security/secl/containerutils/helpers.go +++ b/pkg/security/secl/containerutils/helpers.go @@ -29,39 +29,42 @@ func init() { containerIDPattern = regexp.MustCompile(ContainerIDPatternStr) } -func isSystemdCgroup(cgroup CGroupID) bool { - return strings.HasSuffix(string(cgroup), ".service") || strings.HasSuffix(string(cgroup), ".scope") +func isSystemdScope(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".scope") +} + +func isSystemdService(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".service") +} + +func getSystemdCGroupFlags(cgroup CGroupID) uint64 { + if isSystemdScope(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdScope) + } else if isSystemdService(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdService) + } + return 0 } // FindContainerID extracts the first sub string that matches the pattern of a container ID along with the container flags induced from the container runtime prefix func FindContainerID(s CGroupID) (ContainerID, uint64) { match := containerIDPattern.FindIndex([]byte(s)) if match == nil { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - - return "", 0 + return "", getSystemdCGroupFlags(s) } // first, check what's before if match[0] != 0 { previousChar := string(s[match[0]-1]) if strings.ContainsAny(previousChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } // then, check what's after if match[1] < len(s) { nextChar := string(s[match[1]]) if strings.ContainsAny(nextChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } diff --git a/pkg/security/secl/containerutils/helpers_test.go b/pkg/security/secl/containerutils/helpers_test.go index b5474b0df5016..5a2066e0295f2 100644 --- a/pkg/security/secl/containerutils/helpers_test.go +++ b/pkg/security/secl/containerutils/helpers_test.go @@ -63,7 +63,7 @@ func TestFindContainerID(t *testing.T) { { // Some random path which could match garden format input: "/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-f9176c6a-2a34-4ce2-86af-60d16888ed8e.scope", output: "", - flags: CGroupManagerSystemd, + flags: CGroupManagerSystemd | CGroupManager(SystemdScope), }, { // GARDEN with prefix / suffix input: "prefix01234567-0123-4567-890a-bcdesuffix", diff --git a/pkg/security/secl/go.mod b/pkg/security/secl/go.mod index 7a709c2832e48..da9770680ce9c 100644 --- a/pkg/security/secl/go.mod +++ b/pkg/security/secl/go.mod @@ -41,6 +41,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/pkg/security/secl/go.sum b/pkg/security/secl/go.sum index 267d639c9503a..b3daad8d9516f 100644 --- a/pkg/security/secl/go.sum +++ b/pkg/security/secl/go.sum @@ -83,8 +83,8 @@ golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= diff --git a/pkg/security/security_profile/dump/load_controller.go b/pkg/security/security_profile/dump/load_controller.go index ae8767bcfd153..f9bc80cdd3b4f 100644 --- a/pkg/security/security_profile/dump/load_controller.go +++ b/pkg/security/security_profile/dump/load_controller.go @@ -16,6 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/metrics" + "github.com/DataDog/datadog-agent/pkg/security/secl/containerutils" "github.com/DataDog/datadog-agent/pkg/security/secl/model" "github.com/DataDog/datadog-agent/pkg/security/seclog" ) @@ -34,6 +35,7 @@ type ActivityDumpLoadController struct { // eBPF maps activityDumpConfigDefaults *ebpf.Map + activityDumpLoadConfig map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig } // NewActivityDumpLoadController returns a new activity dump load controller @@ -58,7 +60,11 @@ func NewActivityDumpLoadController(adm *ActivityDumpManager) (*ActivityDumpLoadC }, nil } -func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDumpLoadConfig { +func (lc *ActivityDumpLoadController) getDefaultLoadConfigs() (map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig, error) { + if lc.activityDumpLoadConfig != nil { + return lc.activityDumpLoadConfig, nil + } + defaults := NewActivityDumpLoadConfig( lc.adm.config.RuntimeSecurity.ActivityDumpTracedEventTypes, lc.adm.config.RuntimeSecurity.ActivityDumpCgroupDumpTimeout, @@ -68,14 +74,38 @@ func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDump lc.adm.resolvers.TimeResolver, ) defaults.WaitListTimestampRaw = uint64(lc.adm.config.RuntimeSecurity.ActivityDumpCgroupWaitListTimeout) - return defaults + + allDefaultConfigs := map[string]containerutils.CGroupManager{ + containerutils.CGroupManagerDocker.String(): containerutils.CGroupManagerDocker, + containerutils.CGroupManagerPodman.String(): containerutils.CGroupManagerPodman, + containerutils.CGroupManagerCRI.String(): containerutils.CGroupManagerCRI, + containerutils.CGroupManagerCRIO.String(): containerutils.CGroupManagerCRIO, + containerutils.CGroupManagerSystemd.String(): containerutils.CGroupManagerSystemd, + } + defaultConfigs := make(map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig) + for _, cgroupManager := range lc.adm.config.RuntimeSecurity.ActivityDumpCgroupsManagers { + cgroupManager, found := allDefaultConfigs[cgroupManager] + if !found { + return nil, fmt.Errorf("unsupported cgroup manager '%s'", cgroupManager) + } + defaultConfigs[cgroupManager] = defaults + } + lc.activityDumpLoadConfig = defaultConfigs + return defaultConfigs, nil } -// PushCurrentConfig pushes the current load controller config to kernel space -func (lc *ActivityDumpLoadController) PushCurrentConfig() error { +// PushDefaultCurrentConfigs pushes the current load controller configs to kernel space +func (lc *ActivityDumpLoadController) PushDefaultCurrentConfigs() error { + defaultConfigs, err := lc.getDefaultLoadConfigs() + if err != nil { + return err + } + // push default load config values - if err := lc.activityDumpConfigDefaults.Put(uint32(0), lc.getDefaultLoadConfig()); err != nil { - return fmt.Errorf("couldn't update default activity dump load config: %w", err) + for cgroupManager, defaultConfig := range defaultConfigs { + if err := lc.activityDumpConfigDefaults.Put(uint32(cgroupManager), defaultConfig); err != nil { + return fmt.Errorf("couldn't update default activity dump load config: %w", err) + } } return nil } diff --git a/pkg/security/security_profile/dump/manager.go b/pkg/security/security_profile/dump/manager.go index d06600ed81bca..be81a28ca5c4d 100644 --- a/pkg/security/security_profile/dump/manager.go +++ b/pkg/security/security_profile/dump/manager.go @@ -321,7 +321,8 @@ func NewActivityDumpManager(config *config.Config, statsdClient statsd.ClientInt if err != nil { return nil, fmt.Errorf("couldn't instantiate the activity dump load controller: %w", err) } - if err = loadController.PushCurrentConfig(); err != nil { + + if err = loadController.PushDefaultCurrentConfigs(); err != nil { return nil, fmt.Errorf("failed to push load controller config settings to kernel space: %w", err) } adm.loadController = loadController @@ -449,8 +450,8 @@ func (adm *ActivityDumpManager) HandleCGroupTracingEvent(event *model.CgroupTrac adm.Lock() defer adm.Unlock() - if len(event.ContainerContext.ContainerID) == 0 { - seclog.Warnf("received a cgroup tracing event with an empty container ID") + if len(event.CGroupContext.CGroupID) == 0 { + seclog.Warnf("received a cgroup tracing event with an empty cgroup ID") return } @@ -514,7 +515,19 @@ workloadLoop: } // if we're still here, we can start tracing this workload - if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *adm.loadController.getDefaultLoadConfig()); err != nil { + defaultConfigs, err := adm.loadController.getDefaultLoadConfigs() + if err != nil { + seclog.Errorf("%v", err) + continue + } + + defaultConfig, found := defaultConfigs[containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags)] + if !found { + seclog.Errorf("Failed to find default activity dump config for %s", containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags).String()) + continue + } + + if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *defaultConfig); err != nil { if !errors.Is(err, unix.E2BIG) { seclog.Debugf("%v", err) break @@ -548,11 +561,24 @@ func (adm *ActivityDumpManager) DumpActivity(params *api.ActivityDumpParams) (*a adm.Lock() defer adm.Unlock() + var dumpDuration time.Duration + var err error + if params.Timeout != "" { + if dumpDuration, err = time.ParseDuration(params.Timeout); err != nil { + return nil, err + } + } else { + dumpDuration = adm.config.RuntimeSecurity.ActivityDumpCgroupDumpTimeout + } + + if params.Storage.LocalStorageDirectory == "" { + params.Storage.LocalStorageDirectory = adm.config.RuntimeSecurity.ActivityDumpLocalStorageDirectory + } + newDump := NewActivityDump(adm, func(ad *ActivityDump) { ad.Metadata.ContainerID = containerutils.ContainerID(params.GetContainerID()) ad.Metadata.CGroupContext.CGroupID = containerutils.CGroupID(params.GetCGroupID()) - dumpDuration, _ := time.ParseDuration(params.Timeout) ad.SetTimeout(dumpDuration) if params.GetDifferentiateArgs() { diff --git a/pkg/security/security_profile/dump/storage_manager.go b/pkg/security/security_profile/dump/storage_manager.go index 454309068fad4..ab0382257b6de 100644 --- a/pkg/security/security_profile/dump/storage_manager.go +++ b/pkg/security/security_profile/dump/storage_manager.go @@ -33,12 +33,15 @@ type ActivityDumpStorage interface { type ActivityDumpStorageManager struct { statsdClient statsd.ClientInterface storages map[config.StorageType]ActivityDumpStorage + isFowarder bool } // NewAgentStorageManager returns a new instance of ActivityDumpStorageManager -func NewAgentStorageManager() (*ActivityDumpStorageManager, error) { +func NewAgentStorageManager(statsdClient statsd.ClientInterface) (*ActivityDumpStorageManager, error) { manager := &ActivityDumpStorageManager{ - storages: make(map[config.StorageType]ActivityDumpStorage), + statsdClient: statsdClient, + storages: make(map[config.StorageType]ActivityDumpStorage), + isFowarder: false, } // create remote storage @@ -54,7 +57,8 @@ func NewAgentStorageManager() (*ActivityDumpStorageManager, error) { // NewAgentCommandStorageManager returns a new instance of ActivityDumpStorageManager func NewAgentCommandStorageManager(cfg *config.Config) (*ActivityDumpStorageManager, error) { manager := &ActivityDumpStorageManager{ - storages: make(map[config.StorageType]ActivityDumpStorage), + storages: make(map[config.StorageType]ActivityDumpStorage), + isFowarder: false, } storage, err := NewActivityDumpLocalStorage(cfg, nil) @@ -78,6 +82,7 @@ func NewActivityDumpStorageManager(cfg *config.Config, statsdClient statsd.Clien manager := &ActivityDumpStorageManager{ storages: make(map[config.StorageType]ActivityDumpStorage), statsdClient: statsdClient, + isFowarder: true, } storage, err := NewActivityDumpLocalStorage(cfg, m) @@ -133,7 +138,7 @@ func (manager *ActivityDumpStorageManager) PersistRaw(requests []config.StorageR } // send dump metric - if manager.statsdClient != nil { + if manager.statsdClient != nil && manager.isFowarder { if size := len(raw.Bytes()); size > 0 { tags := []string{"format:" + request.Format.String(), "storage_type:" + request.Type.String(), fmt.Sprintf("compression:%v", request.Compression)} if err := manager.statsdClient.Count(metrics.MetricActivityDumpSizeInBytes, int64(size), tags, 1.0); err != nil { diff --git a/pkg/security/security_profile/dump/storage_manager_unsupported.go b/pkg/security/security_profile/dump/storage_manager_unsupported.go index 885c1f2030808..3fee25e03615c 100644 --- a/pkg/security/security_profile/dump/storage_manager_unsupported.go +++ b/pkg/security/security_profile/dump/storage_manager_unsupported.go @@ -12,6 +12,8 @@ import ( "bytes" "fmt" + "github.com/DataDog/datadog-go/v5/statsd" + "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/proto/api" ) @@ -28,7 +30,7 @@ func (manager *ActivityDumpStorageManager) PersistRaw(_ []config.StorageRequest, func (manager *ActivityDumpStorageManager) SendTelemetry() {} // NewAgentStorageManager returns a new instance of ActivityDumpStorageManager -func NewAgentStorageManager() (*ActivityDumpStorageManager, error) { +func NewAgentStorageManager(_ statsd.ClientInterface) (*ActivityDumpStorageManager, error) { return nil, fmt.Errorf("the activity dump manager is unsupported on this platform") } diff --git a/pkg/security/serializers/serializers_base.go b/pkg/security/serializers/serializers_base.go index 32f84ace271f8..805e52f8c67cc 100644 --- a/pkg/security/serializers/serializers_base.go +++ b/pkg/security/serializers/serializers_base.go @@ -18,15 +18,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/scrubber" ) -// CGroupContextSerializer serializes a cgroup context to JSON -// easyjson:json -type CGroupContextSerializer struct { - // CGroup ID - ID string `json:"id,omitempty"` - // CGroup manager - Manager string `json:"manager,omitempty"` -} - // ContainerContextSerializer serializes a container context to JSON // easyjson:json type ContainerContextSerializer struct { @@ -213,7 +204,6 @@ type BaseEventSerializer struct { *ExitEventSerializer `json:"exit,omitempty"` *ProcessContextSerializer `json:"process,omitempty"` *ContainerContextSerializer `json:"container,omitempty"` - *CGroupContextSerializer `json:"cgroup,omitempty"` } // TLSContextSerializer defines a tls context serializer diff --git a/pkg/security/serializers/serializers_base_linux_easyjson.go b/pkg/security/serializers/serializers_base_linux_easyjson.go index 743795c140527..8502c05b45842 100644 --- a/pkg/security/serializers/serializers_base_linux_easyjson.go +++ b/pkg/security/serializers/serializers_base_linux_easyjson.go @@ -1777,72 +1777,7 @@ func (v ContainerContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { func (v *ContainerContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers14(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(in *jlexer.Lexer, out *CGroupContextSerializer) { - isTopLevel := in.IsStart() - if in.IsNull() { - if isTopLevel { - in.Consumed() - } - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeFieldName(false) - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "id": - out.ID = string(in.String()) - case "manager": - out.Manager = string(in.String()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') - if isTopLevel { - in.Consumed() - } -} -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(out *jwriter.Writer, in CGroupContextSerializer) { - out.RawByte('{') - first := true - _ = first - if in.ID != "" { - const prefix string = ",\"id\":" - first = false - out.RawString(prefix[1:]) - out.String(string(in.ID)) - } - if in.Manager != "" { - const prefix string = ",\"manager\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - out.String(string(in.Manager)) - } - out.RawByte('}') -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v CGroupContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(w, v) -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *CGroupContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(l, v) -} -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(in *jlexer.Lexer, out *BaseEventSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(in *jlexer.Lexer, out *BaseEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -1855,7 +1790,6 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( out.ExitEventSerializer = new(ExitEventSerializer) out.ProcessContextSerializer = new(ProcessContextSerializer) out.ContainerContextSerializer = new(ContainerContextSerializer) - out.CGroupContextSerializer = new(CGroupContextSerializer) in.Delim('{') for !in.IsDelim('}') { key := in.UnsafeFieldName(false) @@ -1912,16 +1846,6 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( } (*out.ContainerContextSerializer).UnmarshalEasyJSON(in) } - case "cgroup": - if in.IsNull() { - in.Skip() - out.CGroupContextSerializer = nil - } else { - if out.CGroupContextSerializer == nil { - out.CGroupContextSerializer = new(CGroupContextSerializer) - } - (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) - } default: in.SkipRecursive() } @@ -1932,7 +1856,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(out *jwriter.Writer, in BaseEventSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(out *jwriter.Writer, in BaseEventSerializer) { out.RawByte('{') first := true _ = first @@ -1992,29 +1916,19 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( } (*in.ContainerContextSerializer).MarshalEasyJSON(out) } - if in.CGroupContextSerializer != nil { - const prefix string = ",\"cgroup\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - (*in.CGroupContextSerializer).MarshalEasyJSON(out) - } out.RawByte('}') } // MarshalEasyJSON supports easyjson.Marshaler interface func (v BaseEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BaseEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(in *jlexer.Lexer, out *AWSSecurityCredentialsSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(in *jlexer.Lexer, out *AWSSecurityCredentialsSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -2053,7 +1967,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(out *jwriter.Writer, in AWSSecurityCredentialsSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(out *jwriter.Writer, in AWSSecurityCredentialsSerializer) { out.RawByte('{') first := true _ = first @@ -2087,14 +2001,14 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AWSSecurityCredentialsSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AWSSecurityCredentialsSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(in *jlexer.Lexer, out *AWSIMDSEventSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(in *jlexer.Lexer, out *AWSIMDSEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -2135,7 +2049,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(out *jwriter.Writer, in AWSIMDSEventSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(out *jwriter.Writer, in AWSIMDSEventSerializer) { out.RawByte('{') first := true _ = first @@ -2154,10 +2068,10 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AWSIMDSEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AWSIMDSEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(l, v) } diff --git a/pkg/security/serializers/serializers_linux.go b/pkg/security/serializers/serializers_linux.go index 69b1cf0bd225a..4f38767025b47 100644 --- a/pkg/security/serializers/serializers_linux.go +++ b/pkg/security/serializers/serializers_linux.go @@ -81,6 +81,15 @@ type FileSerializer struct { MountOrigin string `json:"mount_origin,omitempty"` } +// CGroupContextSerializer serializes a cgroup context to JSON +// easyjson:json +type CGroupContextSerializer struct { + // CGroup ID + ID string `json:"id,omitempty"` + // CGroup manager + Manager string `json:"manager,omitempty"` +} + // UserContextSerializer serializes a user context to JSON // easyjson:json type UserContextSerializer struct { @@ -622,6 +631,7 @@ type EventSerializer struct { *NetworkContextSerializer `json:"network,omitempty"` *DDContextSerializer `json:"dd,omitempty"` *SecurityProfileContextSerializer `json:"security_profile,omitempty"` + *CGroupContextSerializer `json:"cgroup,omitempty"` *SELinuxEventSerializer `json:"selinux,omitempty"` *BPFEventSerializer `json:"bpf,omitempty"` diff --git a/pkg/security/serializers/serializers_linux_easyjson.go b/pkg/security/serializers/serializers_linux_easyjson.go index ec264f2cb6de4..e61dfb697a062 100644 --- a/pkg/security/serializers/serializers_linux_easyjson.go +++ b/pkg/security/serializers/serializers_linux_easyjson.go @@ -3595,6 +3595,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( out.NetworkContextSerializer = new(NetworkContextSerializer) out.DDContextSerializer = new(DDContextSerializer) out.SecurityProfileContextSerializer = new(SecurityProfileContextSerializer) + out.CGroupContextSerializer = new(CGroupContextSerializer) out.SELinuxEventSerializer = new(SELinuxEventSerializer) out.BPFEventSerializer = new(BPFEventSerializer) out.MMapEventSerializer = new(MMapEventSerializer) @@ -3652,6 +3653,16 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*out.SecurityProfileContextSerializer).UnmarshalEasyJSON(in) } + case "cgroup": + if in.IsNull() { + in.Skip() + out.CGroupContextSerializer = nil + } else { + if out.CGroupContextSerializer == nil { + out.CGroupContextSerializer = new(CGroupContextSerializer) + } + (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) + } case "selinux": if in.IsNull() { in.Skip() @@ -3889,16 +3900,6 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*out.ContainerContextSerializer).UnmarshalEasyJSON(in) } - case "cgroup": - if in.IsNull() { - in.Skip() - out.CGroupContextSerializer = nil - } else { - if out.CGroupContextSerializer == nil { - out.CGroupContextSerializer = new(CGroupContextSerializer) - } - (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) - } default: in.SkipRecursive() } @@ -3939,6 +3940,16 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*in.SecurityProfileContextSerializer).MarshalEasyJSON(out) } + if in.CGroupContextSerializer != nil { + const prefix string = ",\"cgroup\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + (*in.CGroupContextSerializer).MarshalEasyJSON(out) + } if in.SELinuxEventSerializer != nil { const prefix string = ",\"selinux\":" if first { @@ -4180,16 +4191,6 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*in.ContainerContextSerializer).MarshalEasyJSON(out) } - if in.CGroupContextSerializer != nil { - const prefix string = ",\"cgroup\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - (*in.CGroupContextSerializer).MarshalEasyJSON(out) - } out.RawByte('}') } @@ -4663,7 +4664,72 @@ func (v CapsetSerializer) MarshalEasyJSON(w *jwriter.Writer) { func (v *CapsetSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers28(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(in *jlexer.Lexer, out *BindEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(in *jlexer.Lexer, out *CGroupContextSerializer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = string(in.String()) + case "manager": + out.Manager = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(out *jwriter.Writer, in CGroupContextSerializer) { + out.RawByte('{') + first := true + _ = first + if in.ID != "" { + const prefix string = ",\"id\":" + first = false + out.RawString(prefix[1:]) + out.String(string(in.ID)) + } + if in.Manager != "" { + const prefix string = ",\"manager\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Manager)) + } + out.RawByte('}') +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v CGroupContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(w, v) +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *CGroupContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(l, v) +} +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(in *jlexer.Lexer, out *BindEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4696,7 +4762,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(out *jwriter.Writer, in BindEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(out *jwriter.Writer, in BindEventSerializer) { out.RawByte('{') first := true _ = first @@ -4715,14 +4781,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BindEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BindEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(in *jlexer.Lexer, out *BPFProgramSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(in *jlexer.Lexer, out *BPFProgramSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4782,7 +4848,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(out *jwriter.Writer, in BPFProgramSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(out *jwriter.Writer, in BPFProgramSerializer) { out.RawByte('{') first := true _ = first @@ -4846,14 +4912,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFProgramSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFProgramSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(in *jlexer.Lexer, out *BPFMapSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(in *jlexer.Lexer, out *BPFMapSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4886,7 +4952,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(out *jwriter.Writer, in BPFMapSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(out *jwriter.Writer, in BPFMapSerializer) { out.RawByte('{') first := true _ = first @@ -4911,14 +4977,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFMapSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFMapSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(in *jlexer.Lexer, out *BPFEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(in *jlexer.Lexer, out *BPFEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4969,7 +5035,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(out *jwriter.Writer, in BPFEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(out *jwriter.Writer, in BPFEventSerializer) { out.RawByte('{') first := true _ = first @@ -4993,14 +5059,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(in *jlexer.Lexer, out *AnomalyDetectionSyscallEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(in *jlexer.Lexer, out *AnomalyDetectionSyscallEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -5031,7 +5097,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(out *jwriter.Writer, in AnomalyDetectionSyscallEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(out *jwriter.Writer, in AnomalyDetectionSyscallEventSerializer) { out.RawByte('{') first := true _ = first @@ -5045,10 +5111,10 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AnomalyDetectionSyscallEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AnomalyDetectionSyscallEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(l, v) } diff --git a/pkg/security/tests/connect_test.go b/pkg/security/tests/connect_test.go index 1e64fe57e1989..7f56dcfb8956c 100644 --- a/pkg/security/tests/connect_test.go +++ b/pkg/security/tests/connect_test.go @@ -12,7 +12,6 @@ import ( "context" "fmt" "net" - "sync" "testing" "golang.org/x/net/nettest" @@ -50,22 +49,20 @@ func TestConnectEvent(t *testing.T) { } t.Run("connect-af-inet-any-tcp-success", func(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("tcp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("tcp", ":4242", done, started) + + // Wait until the server is listening before attempting to connect + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "tcp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "tcp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET), event.Connect.AddrFamily, "wrong address family") @@ -78,22 +75,19 @@ func TestConnectEvent(t *testing.T) { }) t.Run("connect-af-inet-any-udp-success", func(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("udp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("udp", ":4242", done, started) + + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "udp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "udp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET), event.Connect.AddrFamily, "wrong address family") @@ -110,22 +104,19 @@ func TestConnectEvent(t *testing.T) { t.Skip("IPv6 is not supported") } - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("tcp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("tcp", ":4242", done, started) + + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "tcp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "tcp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET6), event.Connect.AddrFamily, "wrong address family") @@ -142,22 +133,19 @@ func TestConnectEvent(t *testing.T) { t.Skip("IPv6 is not supported") } - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("udp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("udp", ":4242", done, started) + + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "udp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "udp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET6), event.Connect.AddrFamily, "wrong address family") @@ -168,43 +156,47 @@ func TestConnectEvent(t *testing.T) { test.validateConnectSchema(t, event) }) }) - } -func bindAndAcceptConnection(proto, address string, done chan struct{}) error { +func bindAndAcceptConnection(proto, address string, done chan error, started chan struct{}) { switch proto { case "tcp": listener, err := net.Listen(proto, address) if err != nil { - return fmt.Errorf("failed to bind to %s:%s: %w", proto, address, err) + done <- fmt.Errorf("failed to bind to %s:%s: %w", proto, address, err) + return } defer listener.Close() - // Start a goroutine to accept connections continuously - go func() { - for { - c, err := listener.Accept() - if err != nil { - fmt.Printf("accept error: %v\n", err) - return - } - fmt.Println("Connection accepted") - defer c.Close() - } - }() + // Signal that the server is ready to accept connections + close(started) + + c, err := listener.Accept() + if err != nil { + fmt.Printf("accept error: %v\n", err) + done <- err + return + } + defer c.Close() + + fmt.Println("Connection accepted") case "udp", "unixgram": conn, err := net.ListenPacket(proto, address) if err != nil { - return fmt.Errorf("failed to bind to %s:%s: %w", proto, address, err) + done <- err + return } defer conn.Close() + // For packet-based connections, we can signal readiness immediately + close(started) + default: - return fmt.Errorf("unsupported protocol: %s", proto) + done <- fmt.Errorf("unsupported protocol: %s", proto) + return } // Wait for the test to complete before returning - <-done - return nil + done <- nil } diff --git a/pkg/security/tests/constants_test.go b/pkg/security/tests/constants_test.go index f7a156aac2954..8982f6a1ff686 100644 --- a/pkg/security/tests/constants_test.go +++ b/pkg/security/tests/constants_test.go @@ -19,20 +19,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/probe/constantfetch" ) -var BTFHubVsRcPossiblyMissingConstants = []string{ - constantfetch.OffsetNameNFConnStructCTNet, - constantfetch.OffsetNameIoKiocbStructCtx, - constantfetch.OffsetNameMountMntID, -} - -var RCVsFallbackPossiblyMissingConstants = []string{ - constantfetch.OffsetNameIoKiocbStructCtx, - constantfetch.OffsetNameTaskStructPID, - constantfetch.OffsetNameTaskStructPIDLink, - constantfetch.OffsetNameDeviceStructNdNet, - constantfetch.OffsetNameMountMntID, -} - var BTFHubVsFallbackPossiblyMissingConstants = []string{ constantfetch.OffsetNameNFConnStructCTNet, constantfetch.OffsetNameTaskStructPID, @@ -69,40 +55,6 @@ func TestOctogonConstants(t *testing.T) { t.Fatal(err) } - _, secconfig, err := genTestConfigs(dir, testOpts{}) - if err != nil { - t.Fatal(err) - } - - t.Run("rc-vs-fallback", func(t *testing.T) { - checkKernelCompatibility(t, "SLES kernels", func(kv *kernel.Version) bool { - return kv.IsSLESKernel() || (kv.IsAmazonLinux2023Kernel() && (testEnvironment == DockerEnvironment)) - }) - - fallbackFetcher := constantfetch.NewFallbackConstantFetcher(kv) - rcFetcher := constantfetch.NewRuntimeCompilationConstantFetcher(&secconfig.Probe.Config, nil) - - assertConstantsEqual(t, rcFetcher, fallbackFetcher, kv, RCVsFallbackPossiblyMissingConstants) - }) - - t.Run("btfhub-vs-rc", func(t *testing.T) { - checkKernelCompatibility(t, "SLES kernels", func(kv *kernel.Version) bool { - return kv.IsSLESKernel() || (kv.IsAmazonLinux2023Kernel() && (testEnvironment == DockerEnvironment)) - }) - - btfhubFetcher, err := constantfetch.NewBTFHubConstantFetcher(kv) - if err != nil { - t.Skipf("btfhub constant fetcher is not available: %v", err) - } - if !btfhubFetcher.HasConstantsInStore() { - t.Skip("btfhub has no constant for this OS") - } - - rcFetcher := constantfetch.NewRuntimeCompilationConstantFetcher(&secconfig.Probe.Config, nil) - - assertConstantsEqual(t, rcFetcher, btfhubFetcher, kv, BTFHubVsRcPossiblyMissingConstants) - }) - t.Run("btfhub-vs-fallback", func(t *testing.T) { btfhubFetcher, err := constantfetch.NewBTFHubConstantFetcher(kv) if err != nil { @@ -116,17 +68,6 @@ func TestOctogonConstants(t *testing.T) { assertConstantsEqual(t, btfhubFetcher, fallbackFetcher, kv, BTFHubVsFallbackPossiblyMissingConstants) }) - - t.Run("guesser-vs-rc", func(t *testing.T) { - checkKernelCompatibility(t, "SLES kernels", func(kv *kernel.Version) bool { - return kv.IsSLESKernel() || (kv.IsAmazonLinux2023Kernel() && (testEnvironment == DockerEnvironment)) - }) - - rcFetcher := constantfetch.NewRuntimeCompilationConstantFetcher(&secconfig.Probe.Config, nil) - ogFetcher := constantfetch.NewOffsetGuesserFetcher(secconfig.Probe, kv) - - assertConstantContains(t, rcFetcher, ogFetcher, kv, nil) - }) } func getFighterConstants(champion, challenger constantfetch.ConstantFetcher, kv *kernel.Version) (map[string]uint64, map[string]uint64, error) { @@ -177,42 +118,6 @@ func assertConstantsEqual(t *testing.T, champion, challenger constantfetch.Const } } -func assertConstantContains(t *testing.T, champion, challenger constantfetch.ConstantFetcher, kv *kernel.Version, ignoreMissing []string) { - t.Helper() - championConstants, challengerConstants, err := getFighterConstants(champion, challenger, kv) - if err != nil { - t.Error(err) - } - - if len(challengerConstants) == 0 { - t.Errorf("challenger %s has no constant\n", challenger) - } - - // In the case where challenger is missing a constant, we set it to the error sentinel - for _, k := range ignoreMissing { - if _, championPresent := championConstants[k]; !championPresent { - championConstants[k] = constantfetch.ErrorSentinel - } - } - - for k, v := range challengerConstants { - if v == constantfetch.ErrorSentinel { - continue - } - - expected, ok := championConstants[k] - if !ok { - t.Errorf("champion (`%s`) does not contain the expected constant `%s`", champion.String(), k) - } else if v != expected && expected != constantfetch.ErrorSentinel { - t.Errorf("difference between fighters for `%s`: `%s`:%d and `%s`:%d", k, champion.String(), expected, challenger.String(), v) - } - } - - if t.Failed() { - t.Logf("kernel version: %v", kv) - } -} - func getOffsetConstantsFromFetcher(cf constantfetch.ConstantFetcher, kv *kernel.Version) (map[string]uint64, error) { probe.AppendProbeRequestsToFetcher(cf, kv) return cf.FinishAndGetResults() diff --git a/pkg/security/tests/container_test.go b/pkg/security/tests/container_test.go index 7a659481da74b..15d896f1718ef 100644 --- a/pkg/security/tests/container_test.go +++ b/pkg/security/tests/container_test.go @@ -182,3 +182,73 @@ func TestContainerFlagsPodman(t *testing.T) { }) }) } + +func TestContainerVariables(t *testing.T) { + SkipIfNotAvailable(t) + + ruleDefs := []*rules.RuleDefinition{ + { + ID: "test_container_set_variable", + Expression: `container.id != "" && open.file.path == "{{.Root}}/test-open"`, + Actions: []*rules.ActionDefinition{ + { + Set: &rules.SetDefinition{ + Scope: "container", + Value: 1, + Name: "foo", + }, + }, + }, + }, + { + ID: "test_container_check_variable", + Expression: `container.id != "" && open.file.path == "{{.Root}}/test-open2" && ${container.foo} == 1`, + }, + } + test, err := newTestModule(t, nil, ruleDefs) + if err != nil { + t.Fatal(err) + } + defer test.Close() + + testFile, _, err := test.Path("test-open") + if err != nil { + t.Fatal(err) + } + + testFile2, _, err := test.Path("test-open2") + if err != nil { + t.Fatal(err) + } + + dockerWrapper, err := newDockerCmdWrapper(test.Root(), test.Root(), "ubuntu", "") + if err != nil { + t.Skip("Skipping created time in containers tests: Docker not available") + return + } + defer dockerWrapper.stop() + + dockerWrapper.Run(t, "container-variables", func(t *testing.T, _ wrapperType, cmdFunc func(cmd string, args []string, envs []string) *exec.Cmd) { + test.WaitSignal(t, func() error { + cmd := cmdFunc("touch", []string{testFile}, nil) + return cmd.Run() + }, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_container_set_variable") + assertFieldEqual(t, event, "open.file.path", testFile) + assertFieldNotEmpty(t, event, "container.id", "container id shouldn't be empty") + + test.validateOpenSchema(t, event) + }) + + test.WaitSignal(t, func() error { + cmd := cmdFunc("touch", []string{testFile2}, nil) + return cmd.Run() + }, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_container_check_variable") + assertFieldEqual(t, event, "open.file.path", testFile2) + assertFieldNotEmpty(t, event, "container.id", "container id shouldn't be empty") + + test.validateOpenSchema(t, event) + }) + }) +} diff --git a/pkg/security/utils/cgroup.go b/pkg/security/utils/cgroup.go index 5dbbb97c58af1..df59e317006a5 100644 --- a/pkg/security/utils/cgroup.go +++ b/pkg/security/utils/cgroup.go @@ -12,6 +12,7 @@ import ( "bufio" "bytes" "crypto/sha256" + "fmt" "os" "strconv" "strings" @@ -49,6 +50,49 @@ func (cg ControlGroup) GetContainerID() containerutils.ContainerID { return containerutils.ContainerID(id) } +// GetLastProcControlGroups returns the first cgroup membership of the specified task. +func GetLastProcControlGroups(tgid, pid uint32) (ControlGroup, error) { + data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) + if err != nil { + return ControlGroup{}, err + } + + data = bytes.TrimSpace(data) + + index := bytes.LastIndexByte(data, '\n') + if index < 0 { + index = 0 + } else { + index++ // to skip the \n + } + if index >= len(data) { + return ControlGroup{}, fmt.Errorf("invalid cgroup data: %s", data) + } + + lastLine := string(data[index:]) + + idstr, rest, ok := strings.Cut(lastLine, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + id, err := strconv.Atoi(idstr) + if err != nil { + return ControlGroup{}, err + } + + controllers, path, ok := strings.Cut(rest, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + return ControlGroup{ + ID: id, + Controllers: strings.Split(controllers, ","), + Path: path, + }, nil +} + // GetProcControlGroups returns the cgroup membership of the specified task. func GetProcControlGroups(tgid, pid uint32) ([]ControlGroup, error) { data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) @@ -85,15 +129,14 @@ func GetProcContainerID(tgid, pid uint32) (containerutils.ContainerID, error) { // GetProcContainerContext returns the container ID which the process belongs to along with its manager. Returns "" if the process does not belong // to a container. func GetProcContainerContext(tgid, pid uint32) (containerutils.ContainerID, model.CGroupContext, error) { - cgroups, err := GetProcControlGroups(tgid, pid) - if err != nil || len(cgroups) == 0 { + cgroup, err := GetLastProcControlGroups(tgid, pid) + if err != nil { return "", model.CGroupContext{}, err } - lastCgroup := len(cgroups) - 1 - containerID, runtime := cgroups[lastCgroup].GetContainerContext() + containerID, runtime := cgroup.GetContainerContext() cgroupContext := model.CGroupContext{ - CGroupID: containerutils.CGroupID(cgroups[lastCgroup].Path), + CGroupID: containerutils.CGroupID(cgroup.Path), CGroupFlags: runtime, } diff --git a/pkg/security/utils/proc_linux.go b/pkg/security/utils/proc_linux.go index 1ee6765ae737e..229a4b16350ed 100644 --- a/pkg/security/utils/proc_linux.go +++ b/pkg/security/utils/proc_linux.go @@ -110,6 +110,11 @@ func StatusPath(pid uint32) string { return procPidPath(pid, "status") } +// TaskStatusPath returns the path to the status file of a task pid in /proc +func TaskStatusPath(pid uint32, task string) string { + return procPidPath3(pid, "task", task, "status") +} + // LoginUIDPath returns the path to the loginuid file of a pid in /proc func LoginUIDPath(pid uint32) string { return procPidPath(pid, "loginuid") @@ -144,6 +149,10 @@ func procPidPath2(pid uint32, path1 string, path2 string) string { return filepath.Join(kernel.ProcFSRoot(), strconv.FormatUint(uint64(pid), 10), path1, path2) } +func procPidPath3(pid uint32, path1, path2, path3 string) string { + return filepath.Join(kernel.ProcFSRoot(), strconv.FormatUint(uint64(pid), 10), path1, path2, path3) +} + // ModulesPath returns the path to the modules file in /proc func ModulesPath() string { return filepath.Join(kernel.ProcFSRoot(), "modules") @@ -410,8 +419,8 @@ func GetProcessPidNamespace(pid uint32) (uint64, error) { } // GetNsPids returns the namespaced pids of the the givent root pid -func GetNsPids(pid uint32) ([]uint32, error) { - statusFile := StatusPath(pid) +func GetNsPids(pid uint32, task string) ([]uint32, error) { + statusFile := TaskStatusPath(pid, task) content, err := os.ReadFile(statusFile) if err != nil { return nil, fmt.Errorf("failed to read status file: %w", err) @@ -442,6 +451,26 @@ func GetNsPids(pid uint32) ([]uint32, error) { return nil, fmt.Errorf("NSpid field not found") } +// GetPidTasks returns the task IDs of a process +func GetPidTasks(pid uint32) ([]string, error) { + taskPath := procPidPath(pid, "task") + + // Read the contents of the task directory + tasks, err := os.ReadDir(taskPath) + if err != nil { + return nil, fmt.Errorf("failed to read task directory: %v", err) + } + + // Collect all task IDs + var taskIDs []string + for _, task := range tasks { + if task.IsDir() { + taskIDs = append(taskIDs, task.Name()) + } + } + return taskIDs, nil +} + // FindPidNamespace search and return the host PID for the given namespaced PID + its namespace func FindPidNamespace(nspid uint32, ns uint64) (uint32, error) { procPids, err := process.Pids() @@ -455,8 +484,17 @@ func FindPidNamespace(nspid uint32, ns uint64) (uint32, error) { continue } - if procNs == ns { - nspids, err := GetNsPids(uint32(procPid)) + if procNs != ns { + continue + } + + tasks, err := GetPidTasks(uint32(procPid)) + if err != nil { + continue + } + + for _, task := range tasks { + nspids, err := GetNsPids(uint32(procPid), task) if err != nil { return 0, err } @@ -529,22 +567,14 @@ var isNsPidAvailable = sync.OnceValue(func() bool { }) // TryToResolveTraceePid tries to resolve and returnt the HOST tracee PID, given the HOST tracer PID and the namespaced tracee PID. -func TryToResolveTraceePid(hostTracerPID, NsTraceePid uint32) (uint32, error) { +func TryToResolveTraceePid(hostTracerPID uint32, tracerNSID uint64, NsTraceePid uint32) (uint32, error) { // Look if the NSpid status field is available or not (it should be, except for Centos7). if isNsPidAvailable() { /* If it's available, we will search for an host pid having the same PID namespace as the tracer, and having the corresponding NS PID in its status field */ - - // 1. get the pid namespace of the tracer - ns, err := GetProcessPidNamespace(hostTracerPID) - if err != nil { - return 0, fmt.Errorf("Failed to resolve PID namespace: %v", err) - } - - // 2. find the host pid matching the arg pid with he tracer namespace - pid, err := FindPidNamespace(NsTraceePid, ns) + pid, err := FindPidNamespace(NsTraceePid, tracerNSID) if err != nil { return 0, fmt.Errorf("Failed to resolve tracee PID namespace: %v", err) } diff --git a/pkg/serializer/go.mod b/pkg/serializer/go.mod index 62d3ab7c2450c..0492d99ac7ba5 100644 --- a/pkg/serializer/go.mod +++ b/pkg/serializer/go.mod @@ -140,7 +140,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -178,7 +178,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/pkg/serializer/go.sum b/pkg/serializer/go.sum index 240d357b106d4..21381032cc3dc 100644 --- a/pkg/serializer/go.sum +++ b/pkg/serializer/go.sum @@ -118,8 +118,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -323,8 +323,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/serializer/internal/metrics/origin_mapping.go b/pkg/serializer/internal/metrics/origin_mapping.go index 45e433d7135b5..02c54e1913363 100644 --- a/pkg/serializer/internal/metrics/origin_mapping.go +++ b/pkg/serializer/internal/metrics/origin_mapping.go @@ -308,6 +308,7 @@ func metricSourceToOriginCategory(ms metrics.MetricSource) int32 { metrics.MetricSourceZk, metrics.MetricSourceAwsNeuron, metrics.MetricSourceNvidiaNim, + metrics.MetricSourceQuarkus, metrics.MetricSourceMilvus: return 11 // integrationMetrics default: @@ -902,6 +903,8 @@ func metricSourceToOriginService(ms metrics.MetricSource) int32 { return 425 case metrics.MetricSourceNvidiaNim: return 426 + case metrics.MetricSourceQuarkus: + return 427 default: return 0 } diff --git a/pkg/serializer/internal/metrics/service_checks_test.go b/pkg/serializer/internal/metrics/service_checks_test.go index 2a96f3d10072a..2559d7c9ed2cc 100644 --- a/pkg/serializer/internal/metrics/service_checks_test.go +++ b/pkg/serializer/internal/metrics/service_checks_test.go @@ -126,7 +126,7 @@ func TestPayloadsEmptyServiceCheck(t *testing.T) { func TestPayloadsServiceChecks(t *testing.T) { config := mock.New(t) - config.Set("serializer_max_payload_size", 200, pkgconfigmodel.SourceAgentRuntime) + config.Set("serializer_max_payload_size", 250, pkgconfigmodel.SourceAgentRuntime) serviceCheckCollection := []ServiceChecks{ {createServiceCheck("1"), createServiceCheck("2"), createServiceCheck("3")}, diff --git a/pkg/serverless/daemon/routes.go b/pkg/serverless/daemon/routes.go index 733b4a49b050c..fe78651ac4d2f 100644 --- a/pkg/serverless/daemon/routes.go +++ b/pkg/serverless/daemon/routes.go @@ -104,7 +104,17 @@ func (e *EndInvocation) ServeHTTP(w http.ResponseWriter, r *http.Request) { } errorMsg := r.Header.Get(invocationlifecycle.InvocationErrorMsgHeader) + if decodedMsg, err := base64.StdEncoding.DecodeString(errorMsg); err != nil { + log.Debug("Error message header may not be encoded, setting as is") + } else { + errorMsg = string(decodedMsg) + } errorType := r.Header.Get(invocationlifecycle.InvocationErrorTypeHeader) + if decodedType, err := base64.StdEncoding.DecodeString(errorType); err != nil { + log.Debug("Error type header may not be encoded, setting as is") + } else { + errorType = string(decodedType) + } errorStack := r.Header.Get(invocationlifecycle.InvocationErrorStackHeader) if decodedStack, err := base64.StdEncoding.DecodeString(errorStack); err != nil { log.Debug("Could not decode error stack header") diff --git a/pkg/serverless/daemon/routes_test.go b/pkg/serverless/daemon/routes_test.go index 25231204af6c8..e630d2829bd61 100644 --- a/pkg/serverless/daemon/routes_test.go +++ b/pkg/serverless/daemon/routes_test.go @@ -7,6 +7,7 @@ package daemon import ( "bytes" + "encoding/base64" "fmt" "io" "net/http" @@ -104,7 +105,7 @@ func TestEndInvocation(t *testing.T) { assert.Equal(m.lastEndDetails.Runtime, d.ExecutionContext.GetCurrentState().Runtime) } -func TestEndInvocationWithError(t *testing.T) { +func TestEndInvocationWithErrorEncodedHeaders(t *testing.T) { assert := assert.New(t) port := testutil.FreeTCPPort(t) d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) @@ -114,10 +115,52 @@ func TestEndInvocationWithError(t *testing.T) { m := &mockLifecycleProcessor{} d.InvocationProcessor = m + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message \n at TestFunction.Handle(ILambdaContext context)" + + client := &http.Client{} + body := bytes.NewBuffer([]byte(`{}`)) + request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) + request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, base64.StdEncoding.EncodeToString([]byte(errorMessage))) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, base64.StdEncoding.EncodeToString([]byte(errorType))) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, base64.StdEncoding.EncodeToString([]byte(errorStack))) + assert.Nil(err) + res, err := client.Do(request) + assert.Nil(err) + if res != nil { + res.Body.Close() + assert.Equal(res.StatusCode, 200) + } + assert.True(m.OnInvokeEndCalled) + assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) +} + +func TestEndInvocationWithErrorNonEncodedHeaders(t *testing.T) { + assert := assert.New(t) + port := testutil.FreeTCPPort(t) + d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) + time.Sleep(100 * time.Millisecond) + defer d.Stop() + + m := &mockLifecycleProcessor{} + d.InvocationProcessor = m + + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message at TestFunction.Handle(ILambdaContext context)" + client := &http.Client{} body := bytes.NewBuffer([]byte(`{}`)) request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, errorMessage) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, errorType) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, errorStack) assert.Nil(err) res, err := client.Do(request) assert.Nil(err) @@ -127,6 +170,9 @@ func TestEndInvocationWithError(t *testing.T) { } assert.True(m.OnInvokeEndCalled) assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) } func TestTraceContext(t *testing.T) { diff --git a/pkg/serverless/logs/logs_test.go b/pkg/serverless/logs/logs_test.go index 6a8f0d30d3cb0..132efb75bbcaa 100644 --- a/pkg/serverless/logs/logs_test.go +++ b/pkg/serverless/logs/logs_test.go @@ -224,7 +224,7 @@ func TestProcessMessageValid(t *testing.T) { lc.processMessage(&message) - received, timed := demux.WaitForNumberOfSamples(7, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(7, 0, 125*time.Millisecond) assert.Len(t, received, 7) assert.Len(t, timed, 0) demux.Reset() @@ -232,7 +232,7 @@ func TestProcessMessageValid(t *testing.T) { lc.enhancedMetricsEnabled = false lc.processMessage(&message) - received, timed = demux.WaitForSamples(100 * time.Millisecond) + received, timed = demux.WaitForSamples(125 * time.Millisecond) assert.Len(t, received, 0, "we should NOT have received metrics") assert.Len(t, timed, 0) } @@ -366,7 +366,7 @@ func TestProcessMessageShouldNotProcessArnNotSet(t *testing.T) { go lc.processMessage(message) - received, timed := demux.WaitForSamples(100 * time.Millisecond) + received, timed := demux.WaitForSamples(125 * time.Millisecond) assert.Len(t, received, 0, "We should NOT have received metrics") assert.Len(t, timed, 0) } @@ -394,7 +394,7 @@ func TestProcessMessageShouldNotProcessLogsDropped(t *testing.T) { go lc.processMessage(message) - received, timed := demux.WaitForSamples(100 * time.Millisecond) + received, timed := demux.WaitForSamples(125 * time.Millisecond) assert.Len(t, received, 0, "We should NOT have received metrics") assert.Len(t, timed, 0) } @@ -424,11 +424,13 @@ func TestProcessMessageShouldProcessLogTypeFunctionOutOfMemory(t *testing.T) { go lc.processMessage(message) - received, timed := demux.WaitForNumberOfSamples(2, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(2, 0, 125*time.Millisecond) assert.Len(t, received, 2) assert.Len(t, timed, 0) - assert.Equal(t, serverlessMetrics.OutOfMemoryMetric, received[0].Name) - assert.Equal(t, serverlessMetrics.ErrorsMetric, received[1].Name) + if len(received) == 2 { + assert.Equal(t, serverlessMetrics.OutOfMemoryMetric, received[0].Name) + assert.Equal(t, serverlessMetrics.ErrorsMetric, received[1].Name) + } } func TestProcessMessageShouldProcessLogTypePlatformReportOutOfMemory(t *testing.T) { @@ -465,7 +467,7 @@ func TestProcessMessageShouldProcessLogTypePlatformReportOutOfMemory(t *testing. go lc.processMessage(message) - received, timed := demux.WaitForNumberOfSamples(2, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(2, 0, 125*time.Millisecond) assert.Len(t, received, 8) assert.Len(t, timed, 0) assert.Equal(t, serverlessMetrics.OutOfMemoryMetric, received[6].Name) @@ -497,7 +499,7 @@ func TestProcessMessageShouldSendFailoverMetric(t *testing.T) { lc.processMessage(&message) - received, timed := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, received, 1) assert.Len(t, timed, 0) demux.Reset() @@ -507,7 +509,7 @@ func TestProcessMessageShouldSendFailoverMetric(t *testing.T) { message.stringRecord = "{\"DD_EXTENSION_FALLBACK_REASON\":\"test-reason\"}" // add again bc processing empties it lc.processMessage(&message) - received, timed = demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + received, timed = demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, received, 1) assert.Len(t, timed, 0) } @@ -549,7 +551,7 @@ func TestProcessLogMessageLogsEnabled(t *testing.T) { assert.NotNil(t, received) assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, "myRequestID", received.Lambda.RequestID) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -597,7 +599,7 @@ func TestProcessLogMessageLogsEnabledForMixedUnorderedMessages(t *testing.T) { assert.NotNil(t, received) assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, expectedRequestIDs[i], received.Lambda.RequestID) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -628,7 +630,7 @@ func TestProcessLogMessageNoStringRecordPlatformLog(t *testing.T) { select { case <-logChannel: assert.Fail(t, "We should not have received logs") - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): // nothing to do here } } @@ -663,7 +665,7 @@ func TestProcessLogMessageNoStringRecordFunctionLog(t *testing.T) { assert.NotNil(t, received) assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, "myRequestID", received.Lambda.RequestID) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -700,7 +702,7 @@ func TestProcessLogMessageLogsNotEnabled(t *testing.T) { select { case <-logChannel: assert.Fail(t, "We should not have received logs") - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): // nothing to do here } } @@ -763,7 +765,7 @@ func TestProcessLogMessagesTimeoutLogFromReportLog(t *testing.T) { assert.Equal(t, "myRequestID", received.Lambda.RequestID) assert.Equal(t, expectedStringRecord[i], string(received.Content)) assert.Equal(t, expectedErrors[i], received.IsError) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -858,7 +860,7 @@ func TestProcessMultipleLogMessagesTimeoutLogFromReportLog(t *testing.T) { } else { assert.False(t, received.IsError) } - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -904,7 +906,7 @@ func TestProcessLogMessagesOutOfMemoryError(t *testing.T) { assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, "myRequestID", received.Lambda.RequestID) assert.Equal(t, true, received.IsError) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -943,7 +945,7 @@ func TestProcessLogMessageLogsNoRequestID(t *testing.T) { select { case <-logChannel: assert.Fail(t, "We should not have received logs") - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): // nothing to do here } @@ -952,7 +954,7 @@ func TestProcessLogMessageLogsNoRequestID(t *testing.T) { for _, msg := range received { assert.Equal(t, "", msg.objectRecord.requestID) } - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -1228,7 +1230,7 @@ func TestRuntimeMetricsMatchLogs(t *testing.T) { lc.processMessage(doneMessage) lc.processMessage(reportMessage) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 125*time.Millisecond) postRuntimeMetricTimestamp := float64(reportLogTime.UnixNano()) / float64(time.Second) runtimeMetricTimestamp := float64(endTime.UnixNano()) / float64(time.Second) assert.Equal(t, generatedMetrics[0], metrics.MetricSample{ @@ -1314,7 +1316,7 @@ func TestRuntimeMetricsMatchLogsProactiveInit(t *testing.T) { lc.processMessage(doneMessage) lc.processMessage(reportMessage) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 125*time.Millisecond) postRuntimeMetricTimestamp := float64(reportLogTime.UnixNano()) / float64(time.Second) runtimeMetricTimestamp := float64(endTime.UnixNano()) / float64(time.Second) assert.Equal(t, generatedMetrics[0], metrics.MetricSample{ @@ -1416,7 +1418,7 @@ func TestRuntimeMetricsOnTimeout(t *testing.T) { assert.Nil(t, restoreErr) lc.processMessage(reportMessage) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 125*time.Millisecond) postRuntimeMetricTimestamp := float64(reportLogTime.UnixNano()) / float64(time.Second) runtimeMetricTimestamp := float64(endTime.UnixNano()) / float64(time.Second) assert.Equal(t, generatedMetrics[0], metrics.MetricSample{ diff --git a/pkg/serverless/metrics/enhanced_metrics_test.go b/pkg/serverless/metrics/enhanced_metrics_test.go index ea6619861f726..9c6f9472ee756 100644 --- a/pkg/serverless/metrics/enhanced_metrics_test.go +++ b/pkg/serverless/metrics/enhanced_metrics_test.go @@ -34,7 +34,7 @@ func TestGenerateEnhancedMetricsFromFunctionLogOutOfMemory(t *testing.T) { GenerateOutOfMemoryEnhancedMetrics(reportLogTime, tags, demux) } - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(2, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(2, 0, 125*time.Millisecond) assert.True(t, isOOM) assert.Len(t, generatedMetrics, 2, "two enhanced metrics should have been generated") assert.Len(t, timedMetrics, 0) @@ -63,7 +63,7 @@ func TestGenerateEnhancedMetricsFromFunctionLogNoMetric(t *testing.T) { GenerateOutOfMemoryEnhancedMetrics(time.Now(), tags, demux) } - generatedMetrics, timedMetrics := demux.WaitForSamples(100 * time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForSamples(125 * time.Millisecond) assert.False(t, isOOM) assert.Len(t, generatedMetrics, 0, "no metrics should have been generated") assert.Len(t, timedMetrics, 0) @@ -89,7 +89,7 @@ func TestGenerateEnhancedMetricsFromReportLogColdStart(t *testing.T) { } go GenerateEnhancedMetricsFromReportLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(7, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(7, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:7], []metrics.MetricSample{{ Name: maxMemoryUsedMetric, @@ -164,7 +164,7 @@ func TestGenerateEnhancedMetricsFromReportLogNoColdStart(t *testing.T) { } go GenerateEnhancedMetricsFromReportLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(6, 0, 0100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(6, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:6], []metrics.MetricSample{{ Name: maxMemoryUsedMetric, @@ -218,7 +218,7 @@ func TestSendTimeoutEnhancedMetric(t *testing.T) { go SendTimeoutEnhancedMetric(tags, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: timeoutsMetric, @@ -238,7 +238,7 @@ func TestSendInvocationEnhancedMetric(t *testing.T) { go SendInvocationEnhancedMetric(tags, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: invocationsMetric, @@ -264,7 +264,7 @@ func TestDisableEnhancedMetrics(t *testing.T) { SendInvocationEnhancedMetric(tags, demux) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -279,7 +279,7 @@ func TestSendOutOfMemoryEnhancedMetric(t *testing.T) { mockTime := time.Now() go SendOutOfMemoryEnhancedMetric(tags, mockTime, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: OutOfMemoryMetric, @@ -298,7 +298,7 @@ func TestSendErrorsEnhancedMetric(t *testing.T) { mockTime := time.Now() go SendErrorsEnhancedMetric(tags, mockTime, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: ErrorsMetric, @@ -366,7 +366,7 @@ func TestGenerateEnhancedMetricsFromRuntimeDoneLogNoStartDate(t *testing.T) { Demux: demux, } go GenerateEnhancedMetricsFromRuntimeDoneLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics, []metrics.MetricSample{{ Name: responseLatencyMetric, Value: 19, @@ -407,7 +407,7 @@ func TestGenerateEnhancedMetricsFromRuntimeDoneLogNoEndDate(t *testing.T) { Demux: demux, } go GenerateEnhancedMetricsFromRuntimeDoneLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics, []metrics.MetricSample{{ Name: responseLatencyMetric, Value: 19, @@ -448,7 +448,7 @@ func TestGenerateEnhancedMetricsFromRuntimeDoneLogOK(t *testing.T) { Demux: demux, } go GenerateEnhancedMetricsFromRuntimeDoneLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics, []metrics.MetricSample{{ Name: runtimeDurationMetric, Value: 153, @@ -487,7 +487,7 @@ func TestGenerateCPUEnhancedMetrics(t *testing.T) { now := float64(time.Now().UnixNano()) / float64(time.Second) args := generateCPUEnhancedMetricsArgs{100, 53, 200, tags, demux, now} go generateCPUEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: cpuSystemTimeMetric, @@ -530,7 +530,7 @@ func TestDisableCPUEnhancedMetrics(t *testing.T) { SendCPUEnhancedMetrics(&proc.CPUData{}, 0, tags, demux) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -560,7 +560,7 @@ func TestGenerateCPUUtilizationEnhancedMetrics(t *testing.T) { Time: now, } go GenerateCPUUtilizationEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(5, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(5, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: cpuTotalUtilizationPctMetric, @@ -621,7 +621,7 @@ func TestGenerateNetworkEnhancedMetrics(t *testing.T) { Time: now, } go generateNetworkEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: rxBytesMetric, @@ -664,7 +664,7 @@ func TestNetworkEnhancedMetricsDisabled(t *testing.T) { SendNetworkEnhancedMetrics(&proc.NetworkData{}, tags, demux) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -685,7 +685,7 @@ func TestSendTmpEnhancedMetrics(t *testing.T) { Time: now, } go generateTmpEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: tmpUsedMetric, @@ -722,7 +722,7 @@ func TestSendTmpEnhancedMetricsDisabled(t *testing.T) { SendTmpEnhancedMetrics(make(chan bool), tags, &metricAgent) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -743,7 +743,7 @@ func TestSendFdEnhancedMetrics(t *testing.T) { Time: now, } go generateFdEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: fdMaxMetric, @@ -779,7 +779,7 @@ func TestSendThreadEnhancedMetrics(t *testing.T) { Time: now, } go generateThreadEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: threadsMaxMetric, @@ -816,7 +816,7 @@ func TestSendProcessEnhancedMetricsDisabled(t *testing.T) { SendProcessEnhancedMetrics(make(chan bool), tags, &metricAgent) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -829,7 +829,7 @@ func TestSendFailoverReasonMetric(t *testing.T) { demux := createDemultiplexer(t) tags := []string{"reason:test-reason"} go SendFailoverReasonMetric(tags, demux) - generatedMetrics, _ := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, _ := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 1) } diff --git a/pkg/serverless/proxy/proxy.go b/pkg/serverless/proxy/proxy.go index 7d43dd36db257..1f8a2e3087658 100644 --- a/pkg/serverless/proxy/proxy.go +++ b/pkg/serverless/proxy/proxy.go @@ -8,6 +8,7 @@ package proxy import ( "context" + "errors" "net/http" "net/http/httputil" "net/url" @@ -68,9 +69,44 @@ func newProxy(target string, processor invocationlifecycle.InvocationProcessor) Scheme: "http", Host: target, } + proxy := httputil.NewSingleHostReverseProxy(url) + + // The default error handler logs "http: proxy error: %v" then returns an HTTP 502 (bad gateway) + // response. This is unfortunate because it lacks much any context on the original request that + // failed, and the commonly observed error today is "context deadline exceeded", which is not + // actionnable if you don't know what request it was for. It also logs to STDERR and does not + // honor the agent's log level. + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + log.Debugf( + "[serverless/proxy][%T] %s %s -- proxy error: %v", + // The dynamic type of processor informs about what kind of proxy this was (main/appsec) + processor, + // The request method and URL are useful to understand what exactly failed. We won't log + // the body (too large) or headers (risks containing sensitive data, such as API keys) + r.Method, r.URL, + // What happened that caused us to be called? + err, + ) + + // If the error is a [context.DeadlineExceeded], we return an HTTP 504 (gateway timeout) + // instead of the generic HTTP 502 (bad gateway) to give the client a better idea of what is + // going on (this may influence retry behavior, for example). + if errors.Is(err, context.DeadlineExceeded) { + w.WriteHeader(http.StatusGatewayTimeout) + } else { + // Return an HTTP 502 (bad gateway) error response; defer the retrying to the client. + w.WriteHeader(http.StatusBadGateway) + } + + // Writing the error message as best-effort, we simply debug-log any error that occur here. + if _, err := w.Write([]byte(err.Error())); err != nil { + log.Debugf("[serverless/proxy][%T] failed to write error message to response body: %v", processor, err) + } + } + return &runtimeProxy{ target: url, - proxy: httputil.NewSingleHostReverseProxy(url), + proxy: proxy, processor: processor, } } diff --git a/pkg/snmp/snmp.go b/pkg/snmp/snmp.go index 8e63ad76a6d3e..db35905dbabf5 100644 --- a/pkg/snmp/snmp.go +++ b/pkg/snmp/snmp.go @@ -95,6 +95,9 @@ type intOrBoolPtr interface { *int | *bool } +// ErrNoConfigGiven is returned when the SNMP listener config was not found +var ErrNoConfigGiven = errors.New("no config given for snmp_listener") + // NewListenerConfig parses configuration and returns a built ListenerConfig func NewListenerConfig() (ListenerConfig, error) { var snmpConfig ListenerConfig @@ -114,7 +117,7 @@ func NewListenerConfig() (ListenerConfig, error) { return snmpConfig, err } } else { - return snmpConfig, errors.New("no config given for snmp_listener") + return snmpConfig, ErrNoConfigGiven } if snmpConfig.AllowedFailures == 0 && snmpConfig.AllowedFailuresLegacy != 0 { diff --git a/pkg/status/systemprobe/status.go b/pkg/status/systemprobe/status.go index 533b94e1f3981..c6cb485094974 100644 --- a/pkg/status/systemprobe/status.go +++ b/pkg/status/systemprobe/status.go @@ -3,8 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build process - // Package systemprobe fetch information about the system probe package systemprobe diff --git a/pkg/status/systemprobe/status_unsupported.go b/pkg/status/systemprobe/status_unsupported.go deleted file mode 100644 index 31c2523258659..0000000000000 --- a/pkg/status/systemprobe/status_unsupported.go +++ /dev/null @@ -1,26 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build !process - -// Package systemprobe fetch information about the system probe -package systemprobe - -import ( - "github.com/DataDog/datadog-agent/comp/core/status" - "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" -) - -// GetStatus returns a notice that it is not supported on systems that do not at least build the process agent -func GetStatus(stats map[string]interface{}, _ string) { - stats["systemProbeStats"] = map[string]interface{}{ - "Errors": "System Probe is not supported on this system", - } -} - -// GetProvider returns nil -func GetProvider(_ sysprobeconfig.Component) status.Provider { - return nil -} diff --git a/pkg/trace/api/api.go b/pkg/trace/api/api.go index c74ecbb137fb9..5cb57d2478a27 100644 --- a/pkg/trace/api/api.go +++ b/pkg/trace/api/api.go @@ -141,7 +141,7 @@ func NewHTTPReceiver( } } log.Infof("Receiver configured with %d decoders and a timeout of %dms", semcount, conf.DecoderTimeout) - containerIDProvider := NewIDProvider(conf.ContainerProcRoot) + containerIDProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) telemetryForwarder := NewTelemetryForwarder(conf, containerIDProvider, statsd) return &HTTPReceiver{ Stats: info.NewReceiverStats(), diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index 9a468dc164487..3d43d0dbe3c6e 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/trace/config" @@ -576,7 +577,9 @@ func TestDecodeV05(t *testing.T) { req, err := http.NewRequest("POST", "/v0.5/traces", bytes.NewReader(b)) assert.NoError(err) req.Header.Set(header.ContainerID, "abcdef123789456") - tp, err := decodeTracerPayload(v05, req, NewIDProvider(""), "python", "3.8.1", "1.2.3") + tp, err := decodeTracerPayload(v05, req, NewIDProvider("", func(_ origindetection.OriginInfo) (string, error) { + return "abcdef123789456", nil + }), "python", "3.8.1", "1.2.3") assert.NoError(err) assert.EqualValues(tp, &pb.TracerPayload{ ContainerID: "abcdef123789456", @@ -1039,14 +1042,26 @@ func TestExpvar(t *testing.T) { } c := newTestReceiverConfig() - c.DebugServerPort = 5012 + c.DebugServerPort = 6789 info.InitInfo(c) + + // Starting a TLS httptest server to retrieve tlsCert + ts := httptest.NewTLSServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + tlsConfig := ts.TLS.Clone() + // Setting a client with the proper TLS configuration + client := ts.Client() + ts.Close() + + // Starting Debug Server s := NewDebugServer(c) + s.SetTLSConfig(tlsConfig) + + // Starting the Debug server s.Start() defer s.Stop() - resp, err := http.Get("http://127.0.0.1:5012/debug/vars") - assert.NoError(t, err) + resp, err := client.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", c.DebugServerPort)) + require.NoError(t, err) defer resp.Body.Close() t.Run("read-expvars", func(t *testing.T) { diff --git a/pkg/trace/api/container.go b/pkg/trace/api/container.go index 508277c5abaf9..0c789f9d06ca7 100644 --- a/pkg/trace/api/container.go +++ b/pkg/trace/api/container.go @@ -12,6 +12,7 @@ import ( "net" "net/http" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" ) @@ -28,7 +29,7 @@ type IDProvider interface { type idProvider struct{} // NewIDProvider initializes an IDProvider instance, in non-linux environments the procRoot arg is unused. -func NewIDProvider(_ string) IDProvider { +func NewIDProvider(_ string, _ func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { return &idProvider{} } diff --git a/pkg/trace/api/container_linux.go b/pkg/trace/api/container_linux.go index 2129a6377c97f..6dd12d227894d 100644 --- a/pkg/trace/api/container_linux.go +++ b/pkg/trace/api/container_linux.go @@ -18,6 +18,7 @@ import ( "syscall" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/util/cgroups" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -96,7 +97,7 @@ func (i *noCgroupsProvider) GetContainerID(_ context.Context, h http.Header) str } // NewIDProvider initializes an IDProvider instance using the provided procRoot to perform cgroups lookups in linux environments. -func NewIDProvider(procRoot string) IDProvider { +func NewIDProvider(procRoot string, containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { // taken from pkg/util/containers/metrics/system.collector_linux.go var hostPrefix string if strings.HasPrefix(procRoot, "/host") { @@ -120,10 +121,11 @@ func NewIDProvider(procRoot string) IDProvider { } c := NewCache(1 * time.Minute) return &cgroupIDProvider{ - procRoot: procRoot, - controller: cgroupController, - cache: c, - reader: reader, + procRoot: procRoot, + controller: cgroupController, + cache: c, + reader: reader, + containerIDFromOriginInfo: containerIDFromOriginInfo, } } @@ -131,8 +133,9 @@ type cgroupIDProvider struct { procRoot string controller string // reader is used to retrieve the container ID from its cgroup v2 inode. - reader *cgroups.Reader - cache *Cache + reader *cgroups.Reader + cache *Cache + containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) } // GetContainerID returns the container ID. @@ -157,6 +160,11 @@ func (c *cgroupIDProvider) GetContainerID(ctx context.Context, h http.Header) st return containerID } + // Retrieve container ID from External Data header + if externalData := h.Get(header.ExternalData); externalData != "" { + return c.resolveContainerIDFromExternalData(externalData) + } + return "" } @@ -296,6 +304,27 @@ func (c *cgroupIDProvider) getCachedContainerID(key string, retrievalFunc func() return val, nil } +// resolveContainerIDFromExternalData returns the container ID for the given External Data. +func (c *cgroupIDProvider) resolveContainerIDFromExternalData(rawExternalData string) string { + var generatedContainerID string + + externalData, err := origindetection.ParseExternalData(rawExternalData) + if err != nil { + log.Errorf("Could not parse external data (%s): %v", rawExternalData, err) + return "" + } + generatedContainerID, err = c.containerIDFromOriginInfo(origindetection.OriginInfo{ + ExternalData: externalData, + ProductOrigin: origindetection.ProductOriginAPM, + }) + if err != nil { + log.Errorf("Could not generate container ID from external data (%s): %v", rawExternalData, err) + return "" + } + + return generatedContainerID +} + // The below cache is copied from /pkg/util/containers/v2/metrics/provider/cache.go. It is not // imported to avoid making the datadog-agent module a dependency of the pkg/trace module. The // datadog-agent module contains replace directives which are not inherited by packages that diff --git a/pkg/trace/api/debug_server.go b/pkg/trace/api/debug_server.go index 828d5357330eb..6fd2f39cc011a 100644 --- a/pkg/trace/api/debug_server.go +++ b/pkg/trace/api/debug_server.go @@ -9,6 +9,7 @@ package api import ( "context" + "crypto/tls" "expvar" "fmt" "net" @@ -29,9 +30,10 @@ const ( // DebugServer serves /debug/* endpoints type DebugServer struct { - conf *config.AgentConfig - server *http.Server - mux *http.ServeMux + conf *config.AgentConfig + server *http.Server + mux *http.ServeMux + tlsConfig *tls.Config } // NewDebugServer returns a debug server @@ -53,13 +55,14 @@ func (ds *DebugServer) Start() { WriteTimeout: defaultTimeout, Handler: ds.setupMux(), } - listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", ds.conf.DebugServerPort)) + listener, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(ds.conf.DebugServerPort))) if err != nil { log.Errorf("Error creating debug server listener: %s", err) return } + tlsListener := tls.NewListener(listener, ds.tlsConfig) go func() { - if err := ds.server.Serve(listener); err != nil && err != http.ErrServerClosed { + if err := ds.server.Serve(tlsListener); err != nil && err != http.ErrServerClosed { log.Errorf("Could not start debug server: %s. Debug server disabled.", err) } }() @@ -82,6 +85,11 @@ func (ds *DebugServer) AddRoute(route string, handler http.Handler) { ds.mux.Handle(route, handler) } +// SetTLSConfig adds the provided tls.Config to the internal http.Server +func (ds *DebugServer) SetTLSConfig(config *tls.Config) { + ds.tlsConfig = config +} + func (ds *DebugServer) setupMux() *http.ServeMux { ds.mux.HandleFunc("/debug/pprof/", pprof.Index) ds.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) diff --git a/pkg/trace/api/debugger.go b/pkg/trace/api/debugger.go index 61164f1d3ed02..51816276a3f3b 100644 --- a/pkg/trace/api/debugger.go +++ b/pkg/trace/api/debugger.go @@ -81,7 +81,7 @@ func debuggerErrorHandler(err error) http.Handler { // newDebuggerProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newDebuggerProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/api/evp_proxy.go b/pkg/trace/api/evp_proxy.go index 452d73017aa8d..2795e6304b605 100644 --- a/pkg/trace/api/evp_proxy.go +++ b/pkg/trace/api/evp_proxy.go @@ -86,7 +86,7 @@ func evpProxyForwarder(conf *config.AgentConfig, statsd statsd.ClientInterface) req.Header["X-Forwarded-For"] = nil }, ErrorLog: logger, - Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot), statsd}, + Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo), statsd}, } } diff --git a/pkg/trace/api/internal/header/headers.go b/pkg/trace/api/internal/header/headers.go index 8ae458be834f5..88cb3549b2836 100644 --- a/pkg/trace/api/internal/header/headers.go +++ b/pkg/trace/api/internal/header/headers.go @@ -25,6 +25,16 @@ const ( // * "ci-,in-" LocalData = "Datadog-Entity-ID" + // ExternalData is a list that contain prefixed-items, split by a ','. Current items are: + // * "it-" if the container is an init container. + // * "cn-" for the container name. + // * "pu-" for the pod UID. + // Order does not matter. + // Possible values: + // * "it-false,cn-nginx,pu-3413883c-ac60-44ab-96e0-9e52e4e173e2" + // * "cn-init,pu-cb4aba1d-0129-44f1-9f1b-b4dc5d29a3b3,it-true" + ExternalData = "Datadog-External-Env" + // Lang specifies the name of the header which contains the language from // which the traces originate. Lang = "Datadog-Meta-Lang" diff --git a/pkg/trace/api/otlp.go b/pkg/trace/api/otlp.go index 88491ea9052ec..eb136d267e090 100644 --- a/pkg/trace/api/otlp.go +++ b/pkg/trace/api/otlp.go @@ -9,7 +9,6 @@ import ( "context" "encoding/hex" "fmt" - "github.com/DataDog/datadog-agent/pkg/trace/transform" "math" "net" "net/http" @@ -26,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/trace/sampler" "github.com/DataDog/datadog-agent/pkg/trace/timing" "github.com/DataDog/datadog-agent/pkg/trace/traceutil" + "github.com/DataDog/datadog-agent/pkg/trace/transform" "github.com/DataDog/datadog-go/v5/statsd" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" @@ -102,7 +102,7 @@ func NewOTLPReceiver(out chan<- *Payload, cfg *config.AgentConfig, statsd statsd enableReceiveResourceSpansV2Val = 1.0 } _ = statsd.Gauge("datadog.trace_agent.otlp.enable_receive_resource_spans_v2", enableReceiveResourceSpansV2Val, nil, 1) - return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} + return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} } // Start starts the OTLPReceiver, if any of the servers were configured as active. diff --git a/pkg/trace/api/pipeline_stats.go b/pkg/trace/api/pipeline_stats.go index b6fe3f2cd161c..b8daf12590bf7 100644 --- a/pkg/trace/api/pipeline_stats.go +++ b/pkg/trace/api/pipeline_stats.go @@ -71,7 +71,7 @@ func pipelineStatsErrorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. func newPipelineStatsProxy(conf *config.AgentConfig, urls []*url.URL, apiKeys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { log.Debug("[pipeline_stats] Creating reverse proxy") - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/profiles.go b/pkg/trace/api/profiles.go index 9b43efc94de74..19add3e307daa 100644 --- a/pkg/trace/api/profiles.go +++ b/pkg/trace/api/profiles.go @@ -109,7 +109,7 @@ func errorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. // For more details please see multiTransport. func newProfileProxy(conf *config.AgentConfig, targets []*url.URL, keys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/symdb.go b/pkg/trace/api/symdb.go index 31b96c494ce5f..b238b478eb65c 100644 --- a/pkg/trace/api/symdb.go +++ b/pkg/trace/api/symdb.go @@ -62,7 +62,7 @@ func symDBErrorHandler(err error) http.Handler { // newSymDBProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newSymDBProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getSymDBDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/config/config.go b/pkg/trace/config/config.go index 5d0e59f059e39..cbf0b0ea5c3a3 100644 --- a/pkg/trace/config/config.go +++ b/pkg/trace/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/obfuscate" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/trace/log" @@ -445,6 +446,9 @@ type AgentConfig struct { // ContainerTags ... ContainerTags func(cid string) ([]string, error) `json:"-"` + // ContainerIDFromOriginInfo ... + ContainerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) `json:"-"` + // ContainerProcRoot is the root dir for `proc` info ContainerProcRoot string diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index fa1ca487a3dc8..e0e3ee4906d99 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -52,6 +52,7 @@ require ( ) require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/shirou/gopsutil/v4 v4.24.11 go.opentelemetry.io/collector/component/componenttest v0.115.0 ) @@ -106,16 +107,16 @@ require ( go.opentelemetry.io/otel/trace v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( github.com/DataDog/datadog-agent => ../../ + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../comp/trace/compression/impl-gzip github.com/DataDog/datadog-agent/comp/trace/compression/impl-zstd => ../../comp/trace/compression/impl-zstd diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index 4613790f7cc34..44ab376fcf70f 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -335,8 +335,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -402,8 +402,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/info/info.go b/pkg/trace/info/info.go index df86c40587fdc..4198100ee6a5b 100644 --- a/pkg/trace/info/info.go +++ b/pkg/trace/info/info.go @@ -8,6 +8,7 @@ package info import ( "bytes" + "crypto/tls" "encoding/json" "expvar" // automatically publish `/debug/vars` on HTTP port "fmt" @@ -236,8 +237,9 @@ func getProgramBanner(version string) (string, string) { // If error is nil, means the program is running. // If not, it displays a pretty-printed message anyway (for support) func Info(w io.Writer, conf *config.AgentConfig) error { - url := fmt.Sprintf("http://127.0.0.1:%d/debug/vars", conf.DebugServerPort) - client := http.Client{Timeout: 3 * time.Second} + url := fmt.Sprintf("https://127.0.0.1:%d/debug/vars", conf.DebugServerPort) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := http.Client{Timeout: 3 * time.Second, Transport: tr} resp, err := client.Get(url) if err != nil { // OK, here, we can't even make an http call on the agent port, diff --git a/pkg/trace/info/info_test.go b/pkg/trace/info/info_test.go index 01c369cac403e..25b7cb42b13ff 100644 --- a/pkg/trace/info/info_test.go +++ b/pkg/trace/info/info_test.go @@ -63,7 +63,7 @@ func (h *testServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func testServer(t *testing.T, testFile string) *httptest.Server { t.Helper() - server := httptest.NewServer(&testServerHandler{t: t, testFile: testFile}) + server := httptest.NewTLSServer(&testServerHandler{t: t, testFile: testFile}) t.Logf("test server (serving fake yet valid data) listening on %s", server.URL) return server } @@ -94,7 +94,7 @@ func (h *testServerWarningHandler) ServeHTTP(w http.ResponseWriter, r *http.Requ } func testServerWarning(t *testing.T) *httptest.Server { - server := httptest.NewServer(&testServerWarningHandler{t: t}) + server := httptest.NewTLSServer(&testServerWarningHandler{t: t}) t.Logf("test server (serving data containing worrying values) listening on %s", server.URL) return server } @@ -119,7 +119,7 @@ func (h *testServerErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques } func testServerError(t *testing.T) *httptest.Server { - server := httptest.NewServer(&testServerErrorHandler{t: t}) + server := httptest.NewTLSServer(&testServerErrorHandler{t: t}) t.Logf("test server (serving bad data to trigger errors) listening on %s", server.URL) return server } @@ -331,7 +331,7 @@ func TestError(t *testing.T) { assert.Equal(len(lines[1]), len(lines[2])) assert.Equal("", lines[3]) assert.Regexp(regexp.MustCompile(`^ Error: .*$`), lines[4]) - assert.Equal(fmt.Sprintf(" URL: http://127.0.0.1:%d/debug/vars", port), lines[5]) + assert.Equal(fmt.Sprintf(" URL: https://127.0.0.1:%d/debug/vars", port), lines[5]) assert.Equal("", lines[6]) assert.Equal("", lines[7]) } diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 306e0217b22ad..3f97dcf41808c 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -20,6 +20,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect @@ -76,18 +77,18 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/grpc v1.67.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../../../../comp/otelcol/otlp/components/metricsclient github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ../../../../comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../comp/trace/compression/def diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index 25a9e6fccac3c..abd898f9bed2a 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -196,8 +196,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -245,8 +245,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/cgroups/go.mod b/pkg/util/cgroups/go.mod index 47a3c2c368f5f..0be6406d65503 100644 --- a/pkg/util/cgroups/go.mod +++ b/pkg/util/cgroups/go.mod @@ -31,7 +31,6 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.35.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/cgroups/go.sum b/pkg/util/cgroups/go.sum index 69f8c3e58e44b..d2a0373b5e095 100644 --- a/pkg/util/cgroups/go.sum +++ b/pkg/util/cgroups/go.sum @@ -38,7 +38,5 @@ google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/defaultpaths/go.mod b/pkg/util/defaultpaths/go.mod index ad84aa45db6e9..0f0c498c8154e 100644 --- a/pkg/util/defaultpaths/go.mod +++ b/pkg/util/defaultpaths/go.mod @@ -22,7 +22,7 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/defaultpaths/go.sum b/pkg/util/defaultpaths/go.sum index 767c365bf98df..82beecfd023fb 100644 --- a/pkg/util/defaultpaths/go.sum +++ b/pkg/util/defaultpaths/go.sum @@ -21,7 +21,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/filesystem/go.mod b/pkg/util/filesystem/go.mod index 1a0effc2f07e8..31b6f1fbd4ab0 100644 --- a/pkg/util/filesystem/go.mod +++ b/pkg/util/filesystem/go.mod @@ -12,7 +12,7 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb github.com/shirou/gopsutil/v4 v4.24.11 github.com/stretchr/testify v1.10.0 golang.org/x/sys v0.28.0 @@ -29,7 +29,6 @@ require ( github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/filesystem/go.sum b/pkg/util/filesystem/go.sum index 97cab95a8f162..ebdb96f4beb1d 100644 --- a/pkg/util/filesystem/go.sum +++ b/pkg/util/filesystem/go.sum @@ -7,8 +7,8 @@ github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -36,7 +36,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/flavor/go.mod b/pkg/util/flavor/go.mod index 4b2095a3d95dd..91efb20fa615f 100644 --- a/pkg/util/flavor/go.mod +++ b/pkg/util/flavor/go.mod @@ -62,7 +62,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/util/flavor/go.sum b/pkg/util/flavor/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/util/flavor/go.sum +++ b/pkg/util/flavor/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/util/grpc/auth.go b/pkg/util/grpc/auth.go index 382c4838919a6..fae83d5c22e8d 100644 --- a/pkg/util/grpc/auth.go +++ b/pkg/util/grpc/auth.go @@ -7,6 +7,7 @@ package grpc import ( "context" + "crypto/subtle" "errors" "fmt" @@ -45,7 +46,7 @@ func AuthInterceptor(verifier verifierFunc) grpc_auth.AuthFunc { // using the given token. func StaticAuthInterceptor(token string) grpc_auth.AuthFunc { return AuthInterceptor(func(reqToken string) (interface{}, error) { - if reqToken != token { + if subtle.ConstantTimeCompare([]byte(reqToken), []byte(token)) == 0 { return struct{}{}, errors.New("invalid session token") } diff --git a/pkg/util/grpc/go.mod b/pkg/util/grpc/go.mod index 46b4cce771519..8c98d5f5187ee 100644 --- a/pkg/util/grpc/go.mod +++ b/pkg/util/grpc/go.mod @@ -40,7 +40,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 google.golang.org/grpc v1.67.1 ) @@ -75,7 +75,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/util/grpc/go.sum b/pkg/util/grpc/go.sum index 6b1fd57ab75ad..c91536ab269b2 100644 --- a/pkg/util/grpc/go.sum +++ b/pkg/util/grpc/go.sum @@ -105,8 +105,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -292,8 +292,8 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= diff --git a/pkg/util/hostname/validate/go.mod b/pkg/util/hostname/validate/go.mod index 6ec1ae79a177d..9c1a227476e2c 100644 --- a/pkg/util/hostname/validate/go.mod +++ b/pkg/util/hostname/validate/go.mod @@ -19,7 +19,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/hostname/validate/go.sum b/pkg/util/hostname/validate/go.sum index c57375d7e46df..d45d69118e85e 100644 --- a/pkg/util/hostname/validate/go.sum +++ b/pkg/util/hostname/validate/go.sum @@ -17,7 +17,5 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/http/go.mod b/pkg/util/http/go.mod index fd083ad63e413..56e028a8317d6 100644 --- a/pkg/util/http/go.mod +++ b/pkg/util/http/go.mod @@ -35,7 +35,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 ) require ( @@ -64,7 +64,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/util/http/go.sum b/pkg/util/http/go.sum index 71817661e74a3..1051ef2578f7d 100644 --- a/pkg/util/http/go.sum +++ b/pkg/util/http/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -254,8 +254,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go index 45ea62c6e2d26..79e8a89895beb 100644 --- a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go +++ b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go @@ -228,9 +228,15 @@ func (le *LeaderEngine) EnsureLeaderElectionRuns() error { func (le *LeaderEngine) runLeaderElection() { for { - log.Infof("Starting leader election process for %q...", le.HolderIdentity) - le.leaderElector.Run(le.ctx) - log.Info("Leader election lost") + select { + case <-le.ctx.Done(): + log.Infof("Quitting leader election process: context was cancelled") + return + default: + log.Infof("Starting leader election process for %q...", le.HolderIdentity) + le.leaderElector.Run(le.ctx) + log.Info("Leader election lost") + } } } diff --git a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go index af604ec84e37c..eefce053e2e22 100644 --- a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go +++ b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go @@ -211,7 +211,7 @@ func TestSubscribe(t *testing.T) { coreClient: client.CoreV1(), coordClient: client.CoordinationV1(), leaderMetric: &dummyGauge{}, - lockType: cmLock.ConfigMapsResourceLock, + lockType: tc.lockType, } notif1 := le.Subscribe() @@ -227,6 +227,9 @@ func TestSubscribe(t *testing.T) { le.EnsureLeaderElectionRuns() require.True(t, le.IsLeader()) + err = tc.getTokenFunc(client) + require.NoError(t, err) + counter1, counter2 := 0, 0 for { select { diff --git a/pkg/util/log/go.mod b/pkg/util/log/go.mod index 26f759935f92e..a439a62283f76 100644 --- a/pkg/util/log/go.mod +++ b/pkg/util/log/go.mod @@ -17,7 +17,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/multierr v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/log/go.sum b/pkg/util/log/go.sum index f46b720dc7374..ffca41d546584 100644 --- a/pkg/util/log/go.sum +++ b/pkg/util/log/go.sum @@ -23,7 +23,5 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/log/setup/go.mod b/pkg/util/log/setup/go.mod index 480a4c8b2e4cd..f2c45da509f53 100644 --- a/pkg/util/log/setup/go.mod +++ b/pkg/util/log/setup/go.mod @@ -63,7 +63,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/pkg/util/log/setup/go.sum b/pkg/util/log/setup/go.sum index 3f00397cbc301..7fdf16db5981c 100644 --- a/pkg/util/log/setup/go.sum +++ b/pkg/util/log/setup/go.sum @@ -86,8 +86,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/pkg/util/scrubber/default_test.go b/pkg/util/scrubber/default_test.go index 86134fbc90328..0bb2d721c3983 100644 --- a/pkg/util/scrubber/default_test.go +++ b/pkg/util/scrubber/default_test.go @@ -6,16 +6,13 @@ package scrubber import ( - "encoding/json" "os" "path/filepath" - "reflect" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" ) func assertClean(t *testing.T, contents, cleanContents string) { @@ -26,96 +23,6 @@ func assertClean(t *testing.T, contents, cleanContents string) { assert.Equal(t, strings.TrimSpace(cleanContents), strings.TrimSpace(cleanedString)) } -func TestConfigScrubbedValidYaml(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "conf.yaml") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - - outputConf := filepath.Join(wd, "test", "conf_scrubbed.yaml") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - - cleaned, err := ScrubBytes([]byte(inputConfData)) - require.NoError(t, err) - - // First test that the a scrubbed yaml is still a valid yaml - var out interface{} - err = yaml.Unmarshal(cleaned, &out) - assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") - - // We replace windows line break by linux so the tests pass on every OS - trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) - trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) - - assert.Equal(t, trimmedOutput, trimmedCleaned) -} - -func TestConfigScrubbedYaml(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "conf_multiline.yaml") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - - outputConf := filepath.Join(wd, "test", "conf_multiline_scrubbed.yaml") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - - cleaned, err := ScrubYaml([]byte(inputConfData)) - require.NoError(t, err) - - // First test that the a scrubbed yaml is still a valid yaml - var out interface{} - err = yaml.Unmarshal(cleaned, &out) - assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") - - // We replace windows line break by linux so the tests pass on every OS - trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) - trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) - - assert.Equal(t, trimmedOutput, trimmedCleaned) -} - -func TestConfigScrubbedJson(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "config.json") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - cleaned, err := ScrubJSON([]byte(inputConfData)) - require.NoError(t, err) - // First test that the a scrubbed json is still valid - var actualOutJSON map[string]interface{} - err = json.Unmarshal(cleaned, &actualOutJSON) - assert.NoError(t, err, "Could not load JSON configuration after being scrubbed") - - outputConf := filepath.Join(wd, "test", "config_scrubbed.json") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - var expectedOutJSON map[string]interface{} - err = json.Unmarshal(outputConfData, &expectedOutJSON) - require.NoError(t, err) - assert.Equal(t, reflect.DeepEqual(expectedOutJSON, actualOutJSON), true) -} - -func TestEmptyYaml(t *testing.T) { - cleaned, err := ScrubYaml(nil) - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) - - cleaned, err = ScrubYaml([]byte("")) - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) -} - -func TestEmptyYamlString(t *testing.T) { - cleaned, err := ScrubYamlString("") - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) -} - func TestConfigStripApiKey(t *testing.T) { assertClean(t, `api_key: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb`, @@ -567,39 +474,8 @@ func TestAddStrippedKeys(t *testing.T) { AddStrippedKeys([]string{"foobar"}) assertClean(t, contents, `foobar: "********"`) -} -func TestAddStrippedKeysExceptions(t *testing.T) { - t.Run("single key", func(t *testing.T) { - contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'` - - AddStrippedKeys([]string{"api_key"}) - - scrubbed, err := ScrubYamlString(contents) - require.Nil(t, err) - require.YAMLEq(t, `api_key: '***************************aaaaa'`, scrubbed) - }) - - t.Run("multiple keys", func(t *testing.T) { - contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' -some_other_key: 'bbbb' -app_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacccc' -yet_another_key: 'dddd'` - - keys := []string{"api_key", "some_other_key", "app_key"} - AddStrippedKeys(keys) - - // check that AddStrippedKeys didn't modify the parameter slice - assert.Equal(t, []string{"api_key", "some_other_key", "app_key"}, keys) - - scrubbed, err := ScrubYamlString(contents) - require.Nil(t, err) - expected := `api_key: '***************************aaaaa' -some_other_key: '********' -app_key: '***********************************acccc' -yet_another_key: 'dddd'` - require.YAMLEq(t, expected, scrubbed) - }) + dynamicReplacers = []Replacer{} } func TestAddStrippedKeysNewReplacer(t *testing.T) { @@ -612,6 +488,8 @@ func TestAddStrippedKeysNewReplacer(t *testing.T) { cleaned, err := newScrubber.ScrubBytes([]byte(contents)) require.NoError(t, err) assert.Equal(t, strings.TrimSpace(`foobar: "********"`), strings.TrimSpace(string(cleaned))) + + dynamicReplacers = []Replacer{} } func TestCertConfig(t *testing.T) { @@ -686,33 +564,6 @@ network_devices: log_level: info`) } -func TestConfigFile(t *testing.T) { - cleanedConfigFile := `dd_url: https://app.datadoghq.com - -api_key: "***************************aaaaa" - -proxy: http://user:********@host:port - - - - - - -dogstatsd_port : 8125 - - -log_level: info -` - - wd, _ := os.Getwd() - filePath := filepath.Join(wd, "test", "datadog.yaml") - cleaned, err := ScrubFile(filePath) - assert.NoError(t, err) - cleanedString := string(cleaned) - - assert.Equal(t, cleanedConfigFile, cleanedString) -} - func TestBearerToken(t *testing.T) { assertClean(t, `Bearer 2fe663014abcd1850076f6d68c0355666db98758262870811cace007cd4a62ba`, @@ -785,3 +636,30 @@ func TestScrubCommandsEnv(t *testing.T) { }) } } + +func TestConfigFile(t *testing.T) { + cleanedConfigFile := `dd_url: https://app.datadoghq.com + +api_key: "***************************aaaaa" + +proxy: http://user:********@host:port + + + + + + +dogstatsd_port : 8125 + + +log_level: info +` + + wd, _ := os.Getwd() + filePath := filepath.Join(wd, "test", "datadog.yaml") + cleaned, err := ScrubFile(filePath) + assert.NoError(t, err) + cleanedString := string(cleaned) + + assert.Equal(t, cleanedConfigFile, cleanedString) +} diff --git a/pkg/util/scrubber/go.mod b/pkg/util/scrubber/go.mod index 6f10e986954d3..ff0eb09afc0ef 100644 --- a/pkg/util/scrubber/go.mod +++ b/pkg/util/scrubber/go.mod @@ -5,14 +5,13 @@ go 1.22.0 require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/stretchr/testify v1.10.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/scrubber/go.sum b/pkg/util/scrubber/go.sum index def44d1759f5c..878d5c29e2e5d 100644 --- a/pkg/util/scrubber/go.sum +++ b/pkg/util/scrubber/go.sum @@ -14,7 +14,5 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/scrubber/json_scrubber.go b/pkg/util/scrubber/json_scrubber.go index 34c65dd357cfa..dbf624b119533 100644 --- a/pkg/util/scrubber/json_scrubber.go +++ b/pkg/util/scrubber/json_scrubber.go @@ -22,7 +22,7 @@ func (c *Scrubber) ScrubJSON(input []byte) ([]byte, error) { if len(input) != 0 && err == nil { c.ScrubDataObj(data) - newInput, err := json.Marshal(data) + newInput, err := json.MarshalIndent(data, "", " ") if err == nil { return newInput, nil } diff --git a/pkg/util/scrubber/json_scrubber_test.go b/pkg/util/scrubber/json_scrubber_test.go index 007da8d3f34cb..b45d0a8d08f78 100644 --- a/pkg/util/scrubber/json_scrubber_test.go +++ b/pkg/util/scrubber/json_scrubber_test.go @@ -6,9 +6,14 @@ package scrubber import ( + "encoding/json" + "os" + "path/filepath" + "reflect" "regexp" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -68,3 +73,25 @@ func TestScrubJSON(t *testing.T) { require.Equal(t, expected, string(actual)) }) } + +func TestConfigScrubbedJson(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "config.json") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + cleaned, err := ScrubJSON([]byte(inputConfData)) + require.NoError(t, err) + // First test that the a scrubbed json is still valid + var actualOutJSON map[string]interface{} + err = json.Unmarshal(cleaned, &actualOutJSON) + assert.NoError(t, err, "Could not load JSON configuration after being scrubbed") + + outputConf := filepath.Join(wd, "test", "config_scrubbed.json") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + var expectedOutJSON map[string]interface{} + err = json.Unmarshal(outputConfData, &expectedOutJSON) + require.NoError(t, err) + assert.Equal(t, reflect.DeepEqual(expectedOutJSON, actualOutJSON), true) +} diff --git a/pkg/util/scrubber/scrubber_test.go b/pkg/util/scrubber/scrubber_test.go index dc23ec52d1971..5a6245d81cddd 100644 --- a/pkg/util/scrubber/scrubber_test.go +++ b/pkg/util/scrubber/scrubber_test.go @@ -28,6 +28,16 @@ func TestNewWithDefaults(t *testing.T) { assert.Equal(t, len(scrubberEmpty.multiLineReplacers), len(scrubber.multiLineReplacers)) } +func TestLastUpdated(t *testing.T) { + scrubber := NewWithDefaults() + for _, replacer := range scrubber.singleLineReplacers { + assert.NotNil(t, replacer.LastUpdated, "single line replacer has no LastUpdated: %v", replacer) + } + for _, replacer := range scrubber.multiLineReplacers { + assert.NotNil(t, replacer.LastUpdated, "multi line replacer has no LastUpdated: %v", replacer) + } +} + func TestRepl(t *testing.T) { scrubber := New() scrubber.AddReplacer(SingleLine, Replacer{ diff --git a/pkg/util/scrubber/test/conf.yaml b/pkg/util/scrubber/test/conf.yaml index cf7fc3d34d3f5..d2bb11a2a9fe3 100644 --- a/pkg/util/scrubber/test/conf.yaml +++ b/pkg/util/scrubber/test/conf.yaml @@ -14,7 +14,7 @@ list_api_key: [aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, aaaaaaaaaaaaaaaaaaaaaaaaaaaabbb list_app_key: [aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb] dict_api_key: {one: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} -dict_app_key: {two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} +dict_app_key: {one: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} additional_endpoints: "https://app.datadoghq.com": @@ -29,15 +29,15 @@ random_url_key1: http://user:p@ssw0r)@host:port random_url_key2: protocol://user:p@ssw0r)@host:port random_url_key3: "http://user:password@host:port" random_domain_key: 'user:password@host:port' -random_url_key1: | +random_url_key4: | http://user:password@host:port -random_url_key2: > +random_url_key5: > http://user:password@host:port -random_url_key3: 'http://user:password@host:port' -random_url_key4: 'mongodb+s.r-v://user:password@host:port' -random_url_key5: 'mongodb+srv://user:pass-with-hyphen@abc.example.com/database' -random_url_key6: 'http://user-with-hyphen:pass-with-hyphen@abc.example.com/database' -random_url_key7: 'http://user-with-hyphen:pass@abc.example.com/database' +random_url_key6: 'http://user:password@host:port' +random_url_key7: 'mongodb+s.r-v://user:password@host:port' +random_url_key8: 'mongodb+srv://user:pass-with-hyphen@abc.example.com/database' +random_url_key9: 'http://user-with-hyphen:pass-with-hyphen@abc.example.com/database' +random_url_key10: 'http://user-with-hyphen:pass@abc.example.com/database' mysql_password1: password mysql_password3: "password" diff --git a/pkg/util/scrubber/test/conf_scrubbed.yaml b/pkg/util/scrubber/test/conf_scrubbed.yaml index c4c987bba5df4..5bb34c750ac3c 100644 --- a/pkg/util/scrubber/test/conf_scrubbed.yaml +++ b/pkg/util/scrubber/test/conf_scrubbed.yaml @@ -14,7 +14,7 @@ list_api_key: ["***************************abbbb", "***************************a list_app_key: ["***********************************abbbb", "***********************************abbbb"] dict_api_key: {one: "***************************abbbb", two: "***************************abbbb"} -dict_app_key: {two: "***********************************abbbb", two: "***********************************abbbb"} +dict_app_key: {one: "***********************************abbbb", two: "***********************************abbbb"} additional_endpoints: "https://app.datadoghq.com": @@ -29,15 +29,15 @@ random_url_key1: http://user:********@host:port random_url_key2: protocol://user:********@host:port random_url_key3: "http://user:********@host:port" random_domain_key: 'user:********@host:port' -random_url_key1: | +random_url_key4: | http://user:********@host:port -random_url_key2: > +random_url_key5: > http://user:********@host:port -random_url_key3: 'http://user:********@host:port' -random_url_key4: 'mongodb+s.r-v://user:********@host:port' -random_url_key5: 'mongodb+srv://user:********@abc.example.com/database' -random_url_key6: 'http://user-with-hyphen:********@abc.example.com/database' -random_url_key7: 'http://user-with-hyphen:********@abc.example.com/database' +random_url_key6: 'http://user:********@host:port' +random_url_key7: 'mongodb+s.r-v://user:********@host:port' +random_url_key8: 'mongodb+srv://user:********@abc.example.com/database' +random_url_key9: 'http://user-with-hyphen:********@abc.example.com/database' +random_url_key10: 'http://user-with-hyphen:********@abc.example.com/database' mysql_password1: "********" mysql_password3: "********" diff --git a/pkg/util/scrubber/yaml_scrubber.go b/pkg/util/scrubber/yaml_scrubber.go index f01d93e064ec5..90f144c6617eb 100644 --- a/pkg/util/scrubber/yaml_scrubber.go +++ b/pkg/util/scrubber/yaml_scrubber.go @@ -6,10 +6,11 @@ package scrubber import ( + "bytes" "fmt" "os" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type scrubCallback = func(string, interface{}) (bool, interface{}) @@ -110,13 +111,16 @@ func (c *Scrubber) ScrubYaml(input []byte) ([]byte, error) { // if we can't load the yaml run the default scrubber on the input if len(input) != 0 && err == nil { c.ScrubDataObj(data) - newInput, err := yaml.Marshal(data) - if err == nil { - input = newInput - } else { - // Since the scrubber is a dependency of the logger we can use it here. + + var buffer bytes.Buffer + encoder := yaml.NewEncoder(&buffer) + encoder.SetIndent(2) + if err := encoder.Encode(&data); err != nil { fmt.Fprintf(os.Stderr, "error scrubbing YAML, falling back on text scrubber: %s\n", err) + } else { + input = buffer.Bytes() } + encoder.Close() } return c.ScrubBytes(input) } diff --git a/pkg/util/scrubber/yaml_scrubber_test.go b/pkg/util/scrubber/yaml_scrubber_test.go index 68f45dbcb9aaa..d576472186bd2 100644 --- a/pkg/util/scrubber/yaml_scrubber_test.go +++ b/pkg/util/scrubber/yaml_scrubber_test.go @@ -6,9 +6,15 @@ package scrubber import ( + "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "gopkg.in/yaml.v3" ) func TestScrubDataObj(t *testing.T) { @@ -62,3 +68,104 @@ func TestScrubDataObj(t *testing.T) { }) } } + +func TestConfigScrubbedValidYaml(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "conf.yaml") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + + outputConf := filepath.Join(wd, "test", "conf_scrubbed.yaml") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + + cleaned, err := ScrubBytes([]byte(inputConfData)) + require.NoError(t, err) + + // First test that the a scrubbed yaml is still a valid yaml + var out interface{} + err = yaml.Unmarshal(cleaned, &out) + assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") + + // We replace windows line break by linux so the tests pass on every OS + trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) + trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) + + assert.Equal(t, trimmedOutput, trimmedCleaned) +} + +func TestConfigScrubbedYaml(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "conf_multiline.yaml") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + + outputConf := filepath.Join(wd, "test", "conf_multiline_scrubbed.yaml") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + + cleaned, err := ScrubYaml([]byte(inputConfData)) + require.NoError(t, err) + + // First test that the a scrubbed yaml is still a valid yaml + var out interface{} + err = yaml.Unmarshal(cleaned, &out) + assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") + + // We replace windows line break by linux so the tests pass on every OS + trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) + trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) + + assert.Equal(t, trimmedOutput, trimmedCleaned) +} + +func TestEmptyYaml(t *testing.T) { + cleaned, err := ScrubYaml(nil) + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) + + cleaned, err = ScrubYaml([]byte("")) + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) +} + +func TestEmptyYamlString(t *testing.T) { + cleaned, err := ScrubYamlString("") + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) +} + +func TestAddStrippedKeysExceptions(t *testing.T) { + t.Run("single key", func(t *testing.T) { + contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'` + + AddStrippedKeys([]string{"api_key"}) + + scrubbed, err := ScrubYamlString(contents) + require.Nil(t, err) + require.YAMLEq(t, `api_key: '***************************aaaaa'`, scrubbed) + }) + + t.Run("multiple keys", func(t *testing.T) { + contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +some_other_key: 'bbbb' +app_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacccc' +yet_another_key: 'dddd'` + + keys := []string{"api_key", "some_other_key", "app_key"} + AddStrippedKeys(keys) + + // check that AddStrippedKeys didn't modify the parameter slice + assert.Equal(t, []string{"api_key", "some_other_key", "app_key"}, keys) + + scrubbed, err := ScrubYamlString(contents) + require.Nil(t, err) + expected := `api_key: '***************************aaaaa' +some_other_key: '********' +app_key: '***********************************acccc' +yet_another_key: 'dddd'` + require.YAMLEq(t, expected, scrubbed) + }) +} diff --git a/pkg/util/system/go.mod b/pkg/util/system/go.mod index b91b137b720c2..69183e1155599 100644 --- a/pkg/util/system/go.mod +++ b/pkg/util/system/go.mod @@ -30,14 +30,13 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/ebitengine/purego v0.8.1 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/system/go.sum b/pkg/util/system/go.sum index 5dc55f7f10161..5eac83e6a0bd2 100644 --- a/pkg/util/system/go.sum +++ b/pkg/util/system/go.sum @@ -10,8 +10,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -45,7 +45,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/trivy/trivy.go b/pkg/util/trivy/trivy.go index e78690f98ab6b..63163ff3ae307 100644 --- a/pkg/util/trivy/trivy.go +++ b/pkg/util/trivy/trivy.go @@ -56,7 +56,6 @@ const ( LanguagesAnalyzers = "languages" // LanguagesAnalyzers defines a language analyzer SecretAnalyzers = "secret" // SecretAnalyzers defines a secret analyzer ConfigFileAnalyzers = "config" // ConfigFileAnalyzers defines a configuration file analyzer - LicenseAnalyzers = "license" // LicenseAnalyzers defines a license analyzer TypeApkCommand = "apk-command" // TypeApkCommand defines a apk-command analyzer HistoryDockerfile = "history-dockerfile" // HistoryDockerfile defines a history-dockerfile analyzer TypeImageConfigSecret = "image-config-secret" // TypeImageConfigSecret defines a history-dockerfile analyzer @@ -149,9 +148,6 @@ func DefaultDisabledCollectors(enabledAnalyzers []string) []analyzer.Type { if analyzersDisabled(ConfigFileAnalyzers) { disabledAnalyzers = append(disabledAnalyzers, analyzer.TypeConfigFiles...) } - if analyzersDisabled(LicenseAnalyzers) { - disabledAnalyzers = append(disabledAnalyzers, analyzer.TypeLicenseFile) - } if analyzersDisabled(TypeApkCommand) { disabledAnalyzers = append(disabledAnalyzers, analyzer.TypeApkCommand) } @@ -166,7 +162,9 @@ func DefaultDisabledCollectors(enabledAnalyzers []string) []analyzer.Type { analyzer.TypeRedHatContentManifestType, analyzer.TypeRedHatDockerfileType, analyzer.TypeSBOM, - analyzer.TypeUbuntuESM) + analyzer.TypeUbuntuESM, + analyzer.TypeLicenseFile, + ) return disabledAnalyzers } diff --git a/pkg/util/uuid/go.mod b/pkg/util/uuid/go.mod index ad2e7ef2c7f0a..0f8d47fc3bb8c 100644 --- a/pkg/util/uuid/go.mod +++ b/pkg/util/uuid/go.mod @@ -28,7 +28,7 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/uuid/go.sum b/pkg/util/uuid/go.sum index 2e5f8b7759901..7399205747094 100644 --- a/pkg/util/uuid/go.sum +++ b/pkg/util/uuid/go.sum @@ -44,7 +44,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/winutil/go.mod b/pkg/util/winutil/go.mod index 1facedf93843c..172db988efae7 100644 --- a/pkg/util/winutil/go.mod +++ b/pkg/util/winutil/go.mod @@ -21,7 +21,6 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/winutil/go.sum b/pkg/util/winutil/go.sum index 99e864f633ef6..788135f619bbb 100644 --- a/pkg/util/winutil/go.sum +++ b/pkg/util/winutil/go.sum @@ -21,7 +21,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml b/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml new file mode 100644 index 0000000000000..0e5a6cd7f5c92 --- /dev/null +++ b/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG-DCA.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + The `kubernetes_state_core` check now collects only metadata for configmaps, + reducing memory, CPU, and network usage in the Cluster Agent while preserving + full metric functionality. diff --git a/releasenotes/notes/Add-metrics-origins-for-Quarkus-integration.-e2ff9ac00b7ea6df.yaml b/releasenotes/notes/Add-metrics-origins-for-Quarkus-integration.-e2ff9ac00b7ea6df.yaml new file mode 100644 index 0000000000000..6a23ab8155f8c --- /dev/null +++ b/releasenotes/notes/Add-metrics-origins-for-Quarkus-integration.-e2ff9ac00b7ea6df.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +other: + - | + Add metrics origins for Quarkus integration. diff --git a/releasenotes/notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml b/releasenotes/notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml new file mode 100644 index 0000000000000..9bff569f5f774 --- /dev/null +++ b/releasenotes/notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + On Windows, Agent flares now include event logs for .NET applications. diff --git a/releasenotes/notes/add-integration-field-to-topology-link-metadata-8a35c0a70ac80b1e.yaml b/releasenotes/notes/add-integration-field-to-topology-link-metadata-8a35c0a70ac80b1e.yaml new file mode 100644 index 0000000000000..003ca577e3799 --- /dev/null +++ b/releasenotes/notes/add-integration-field-to-topology-link-metadata-8a35c0a70ac80b1e.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Emit new field: integration for TopologyLinkMetadata for better metrics collection. + diff --git a/releasenotes/notes/add_system_cpu_total_metrics-769eed5d476ba712.yaml b/releasenotes/notes/add_system_cpu_total_metrics-769eed5d476ba712.yaml new file mode 100644 index 0000000000000..9d2899b7bfcc1 --- /dev/null +++ b/releasenotes/notes/add_system_cpu_total_metrics-769eed5d476ba712.yaml @@ -0,0 +1,15 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Add new system.cpu.user.total, system.cpu.nice.total, + system.cpu.system.total, system.cpu.idle.total, system.cpu.iowait.total, + system.cpu.irq.total, system.cpu.softirq.total, system.cpu.steal.total, + system.cpu.guest.total, system.cpu.guestnice.total metrics + with core tag for each one of them. diff --git a/releasenotes/notes/agent-sidecar-security-cbfd5ea9f72124d0.yaml b/releasenotes/notes/agent-sidecar-security-cbfd5ea9f72124d0.yaml new file mode 100644 index 0000000000000..a1f2095a0d121 --- /dev/null +++ b/releasenotes/notes/agent-sidecar-security-cbfd5ea9f72124d0.yaml @@ -0,0 +1,14 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +upgrade: +enhancements: + - | + The Datadog Cluster Agent admission controller agent sidecar injection now sets up + Agent sidecars to run with securityContext of `readOnlyRootFilesystem:false` by default. + Advanced users can customize the securityContext via clusterAgent.admissionController.agentSidecarInjection.profiles. diff --git a/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml new file mode 100644 index 0000000000000..4aa3e9ef23e70 --- /dev/null +++ b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Implement External Data resolution for APM. This is needed to support the + latest Origin Detection spec and resolution with nested virtualization. + diff --git a/releasenotes/notes/fix-oracle-can_connect-host-tag-c7b00cb45e17735c.yaml b/releasenotes/notes/fix-oracle-can_connect-host-tag-c7b00cb45e17735c.yaml new file mode 100644 index 0000000000000..df805c338b08c --- /dev/null +++ b/releasenotes/notes/fix-oracle-can_connect-host-tag-c7b00cb45e17735c.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Corrects host tagging for the oracle.can_connect service check diff --git a/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml new file mode 100644 index 0000000000000..2baa2dea73281 --- /dev/null +++ b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Added the output of ``semodule -l`` to the Agent flare; this information appears in ``system-probe/selinux_semodule_list.log``. diff --git a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml index 53476dcfa0bb9..e9c1031ef5d3b 100644 --- a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml +++ b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml @@ -9,7 +9,7 @@ features: - | Introduced a new configuration variable `logs_config.http_protocol`, allowing users to enforce HTTP/1.1 for outgoing HTTP connections in the Datadog Agent. This provides better control over transport protocols and improves compatibility with systems that do not support HTTP/2. - By default, the log agent will now attempt to use HTTP/2 and fall back to the best available protocol if HTTP/2 is not supported. + By default, the log agent will now attempt to use HTTP/2 (unless a proxy is configured) and fall back to the best available protocol if HTTP/2 is not supported. enhancements: - | - Improved logging to add visiblity for latency and transport protocol \ No newline at end of file + Improved logging to add visibility for latency and transport protocol \ No newline at end of file diff --git a/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml new file mode 100644 index 0000000000000..7d9a8653d8f0d --- /dev/null +++ b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Metric payloads are compressed using `zstd` compression by default. + This can be reverted to the previous compression kind by adding + ``serializer_compressor_kind: zlib`` to the configuration. diff --git a/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml new file mode 100644 index 0000000000000..2957ae7f9441d --- /dev/null +++ b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Adds support for the `none` cardinality type in the tagger. diff --git a/releasenotes/notes/process-agent-config-auth-09a3123ac6496052.yaml b/releasenotes/notes/process-agent-config-auth-09a3123ac6496052.yaml new file mode 100644 index 0000000000000..f95ff5b472dac --- /dev/null +++ b/releasenotes/notes/process-agent-config-auth-09a3123ac6496052.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + All process agent endpoints now require authentication. diff --git a/releasenotes/notes/update-embedded-permissions-a8e169f1c079cd41.yaml b/releasenotes/notes/update-embedded-permissions-a8e169f1c079cd41.yaml new file mode 100644 index 0000000000000..8b9cafd486a16 --- /dev/null +++ b/releasenotes/notes/update-embedded-permissions-a8e169f1c079cd41.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +security: + - | + Move the embedded Python cache out of the installation directory on Windows. diff --git a/tasks/__init__.py b/tasks/__init__.py index d59a06de7463b..0dffed23dacaf 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -87,13 +87,13 @@ tidy, tidy_all, ) +from tasks.gointegrationtest import integration_tests from tasks.gotest import ( check_otel_build, check_otel_module_versions, e2e_tests, get_impacted_packages, get_modified_packages, - integration_tests, lint_go, send_unit_tests_stats, test, diff --git a/tasks/agent.py b/tasks/agent.py index d4e5d13230885..84d1134310965 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -17,6 +17,11 @@ from tasks.build_tags import add_fips_tags, filter_incompatible_tags, get_build_tags, get_default_build_tags from tasks.devcontainer import run_on_devcontainer from tasks.flavor import AgentFlavor +from tasks.gointegrationtest import ( + CORE_AGENT_LINUX_IT_CONF, + CORE_AGENT_WINDOWS_IT_CONF, + containerized_integration_tests, +) from tasks.libs.common.utils import ( REPO_PATH, bin_name, @@ -347,9 +352,8 @@ def refresh_assets(_, build_tags, development=True, flavor=AgentFlavor.base.name check_dir = os.path.join(dist_folder, f"conf.d/{check}.d/") shutil.copytree(f"./cmd/agent/dist/conf.d/{check}.d/", check_dir, dirs_exist_ok=True) - if "apm" in build_tags: + if sys.platform == 'darwin': shutil.copy("./cmd/agent/dist/conf.d/apm.yaml.default", os.path.join(dist_folder, "conf.d/apm.yaml.default")) - if "process" in build_tags: shutil.copy( "./cmd/agent/dist/conf.d/process_agent.yaml.default", os.path.join(dist_folder, "conf.d/process_agent.yaml.default"), @@ -578,79 +582,18 @@ def integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", t """ Run integration tests for the Agent """ - if sys.platform == 'win32': - return _windows_integration_tests(ctx, race=race, go_mod=go_mod, timeout=timeout) - else: - # TODO: See if these will function on Windows - return _linux_integration_tests(ctx, race=race, remote_docker=remote_docker, go_mod=go_mod, timeout=timeout) - - -def _windows_integration_tests(ctx, race=False, go_mod="readonly", timeout=""): - test_args = { - "go_mod": go_mod, - "go_build_tags": " ".join(get_default_build_tags(build="test")), - "race_opt": "-race" if race else "", - "exec_opts": "", - "timeout_opt": f"-timeout {timeout}" if timeout else "", - } - - go_cmd = 'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) # noqa: FS002 - - tests = [ - { - # Run eventlog tests with the Windows API, which depend on the EventLog service - "dir": "./pkg/util/winutil/", - 'prefix': './eventlog/...', - 'extra_args': '-evtapi Windows', - }, - { - # Run eventlog tailer tests with the Windows API, which depend on the EventLog service - "dir": ".", - 'prefix': './pkg/logs/tailers/windowsevent/...', - 'extra_args': '-evtapi Windows', - }, - { - # Run eventlog check tests with the Windows API, which depend on the EventLog service - "dir": ".", - # Don't include submodules, since the `-evtapi` flag is not defined in them - 'prefix': './comp/checks/windowseventlog/windowseventlogimpl/check', - 'extra_args': '-evtapi Windows', - }, - ] - - for test in tests: - with ctx.cd(f"{test['dir']}"): - ctx.run(f"{go_cmd} {test['prefix']} {test['extra_args']}") - - -def _linux_integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", timeout=""): - test_args = { - "go_mod": go_mod, - "go_build_tags": " ".join(get_default_build_tags(build="test")), - "race_opt": "-race" if race else "", - "exec_opts": "", - "timeout_opt": f"-timeout {timeout}" if timeout else "", - } - - # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout - # to the call, we don't want them because while calling invoke below, invoke - # thinks that the parameters are for it to interpret. - # we're calling an intermediate script which only pass the binary name to the invoke task. - if remote_docker: - test_args["exec_opts"] = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" - - go_cmd = 'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) # noqa: FS002 - - prefixes = [ - "./test/integration/config_providers/...", - "./test/integration/corechecks/...", - "./test/integration/listeners/...", - "./test/integration/util/kubelet/...", - ] - - for prefix in prefixes: - ctx.run(f"{go_cmd} {prefix}") + return containerized_integration_tests( + ctx, CORE_AGENT_WINDOWS_IT_CONF, race=race, go_mod=go_mod, timeout=timeout + ) + return containerized_integration_tests( + ctx, + CORE_AGENT_LINUX_IT_CONF, + race=race, + remote_docker=remote_docker, + go_mod=go_mod, + timeout=timeout, + ) def check_supports_python_version(check_dir, python): diff --git a/tasks/build_tags.py b/tasks/build_tags.py index a3c13f088eb8b..9f58d3e45c60a 100644 --- a/tasks/build_tags.py +++ b/tasks/build_tags.py @@ -17,7 +17,6 @@ # ALL_TAGS lists all available build tags. # Used to remove unknown tags from provided tag lists. ALL_TAGS = { - "apm", "clusterchecks", "consul", "containerd", @@ -40,7 +39,6 @@ "otlp", "pcap", # used by system-probe to compile packet filters using google/gopacket/pcap, which requires cgo to link libpcap "podman", - "process", "python", "sds", "serverless", @@ -58,7 +56,6 @@ # AGENT_TAGS lists the tags needed when building the agent. AGENT_TAGS = { - "apm", "consul", "containerd", "cri", @@ -76,7 +73,6 @@ "orchestrator", "otlp", "podman", - "process", "python", "systemd", "trivy", @@ -159,7 +155,17 @@ SERVERLESS_TAGS = {"serverless", "otlp"} # SYSTEM_PROBE_TAGS lists the tags necessary to build system-probe -SYSTEM_PROBE_TAGS = AGENT_TAGS.union({"linux_bpf", "npm", "pcap"}).difference({"python", "systemd"}) +SYSTEM_PROBE_TAGS = { + "datadog.no_waf", + "ec2", + "linux_bpf", + "netcgo", + "npm", + "pcap", + "trivy", + "zlib", + "zstd", +} # TRACE_AGENT_TAGS lists the tags that have to be added when the trace-agent TRACE_AGENT_TAGS = {"docker", "containerd", "datadog.no_waf", "kubeapiserver", "kubelet", "otlp", "netcgo", "podman"} diff --git a/tasks/buildimages.py b/tasks/buildimages.py index 2f7d228d4d070..06ea84211ac1a 100644 --- a/tasks/buildimages.py +++ b/tasks/buildimages.py @@ -1,7 +1,5 @@ from __future__ import annotations -import os - import yaml from invoke import Context, Exit, task @@ -37,17 +35,33 @@ def update(_: Context, tag: str = "", images: str = "", test: bool = True, list_ print(f" {', '.join(modified)}") -@task(help={"commit_sha": "commit sha from the test-infra-definitions repository"}) -def update_test_infra_definitions(ctx: Context, commit_sha: str, go_mod_only: bool = False): +@task( + help={ + "commit_sha": "commit sha from the test-infra-definitions repository", + "go_mod_only": "Update only the go.mod file", + "is_dev_image": "Is the image bumped to a dev version, used to test changes in test-infra-definitions", + } +) +def update_test_infra_definitions(ctx: Context, commit_sha: str, go_mod_only: bool = False, is_dev_image: bool = False): """ Update the test-infra-definition image version in the Gitlab CI as well as in the e2e go.mod """ - if not go_mod_only: - update_test_infra_def(".gitlab/common/test_infra_version.yml", commit_sha[:12]) + print(f"Updating test-infra-definitions to {commit_sha}") + with ctx.cd("test/new-e2e"): + ctx.run(f"go get github.com/DataDog/test-infra-definitions@{commit_sha}") + ctx.run("go mod tidy") - os.chdir("test/new-e2e") - ctx.run(f"go get github.com/DataDog/test-infra-definitions@{commit_sha}") - ctx.run("go mod tidy") + if not go_mod_only: + prefix_comment = """# File generated by inv buildimages.update-test-infra-definitions +# Please do not edit this file manually +# To update the test-infra-definitions version, run `inv buildimages.update-test-infra-definitions --commit-sha ` [--is-dev-image] +""" + update_test_infra_def( + file_path=".gitlab/common/test_infra_version.yml", + image_tag=commit_sha[:12], + is_dev_image=is_dev_image, + prefix_comment=prefix_comment, + ) @task( diff --git a/tasks/cluster_agent.py b/tasks/cluster_agent.py index 8bbde5be51ece..b929270831a58 100644 --- a/tasks/cluster_agent.py +++ b/tasks/cluster_agent.py @@ -12,10 +12,10 @@ from invoke import task from invoke.exceptions import Exit -from tasks.build_tags import get_build_tags, get_default_build_tags +from tasks.build_tags import get_default_build_tags from tasks.cluster_agent_helpers import build_common, clean_common, refresh_assets_common, version_common from tasks.cws_instrumentation import BIN_PATH as CWS_INSTRUMENTATION_BIN_PATH -from tasks.libs.common.utils import TestsNotSupportedError +from tasks.gointegrationtest import CLUSTER_AGENT_IT_CONF, containerized_integration_tests from tasks.libs.releasing.version import load_release_versions # constants @@ -92,33 +92,14 @@ def integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", t """ Run integration tests for cluster-agent """ - if sys.platform == 'win32': - raise TestsNotSupportedError('Cluster Agent integration tests are not supported on Windows') - - # We need docker for the kubeapiserver integration tests - tags = get_default_build_tags(build="cluster-agent") + ["docker", "test"] - - go_build_tags = " ".join(get_build_tags(tags, [])) - race_opt = "-race" if race else "" - exec_opts = "" - timeout_opt = f"-timeout {timeout}" if timeout else "" - - # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout - # to the call, we don't want them because while calling invoke below, invoke - # thinks that the parameters are for it to interpret. - # we're calling an intermediate script which only pass the binary name to the invoke task. - if remote_docker: - exec_opts = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" - - go_cmd = f'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}' - - prefixes = [ - "./test/integration/util/kube_apiserver", - "./test/integration/util/leaderelection", - ] - - for prefix in prefixes: - ctx.run(f"{go_cmd} {prefix}") + containerized_integration_tests( + ctx, + CLUSTER_AGENT_IT_CONF, + race=race, + remote_docker=remote_docker, + go_mod=go_mod, + timeout=timeout, + ) @task diff --git a/tasks/collector.py b/tasks/collector.py index e4dbd4ae4d68d..74f3a439bd9fd 100644 --- a/tasks/collector.py +++ b/tasks/collector.py @@ -14,7 +14,7 @@ from tasks.go import tidy from tasks.libs.ciproviders.github_api import GithubAPI from tasks.libs.common.color import Color, color_message -from tasks.libs.common.git import check_uncommitted_changes +from tasks.libs.common.git import check_uncommitted_changes, get_git_config, revert_git_config, set_git_config LICENSE_HEADER = """// Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. @@ -502,23 +502,6 @@ def update(ctx): print("Update complete.") -def get_git_config(key): - result = subprocess.run(['git', 'config', '--get', key], capture_output=True, text=True) - return result.stdout.strip() if result.returncode == 0 else None - - -def set_git_config(key, value): - subprocess.run(['git', 'config', key, value]) - - -def revert_git_config(original_config): - for key, value in original_config.items(): - if value is None: - subprocess.run(['git', 'config', '--unset', key]) - else: - subprocess.run(['git', 'config', key, value]) - - @task() def pull_request(ctx): # Save current Git configuration diff --git a/tasks/dogstatsd.py b/tasks/dogstatsd.py index 08dbe80a85993..311560a3d118e 100644 --- a/tasks/dogstatsd.py +++ b/tasks/dogstatsd.py @@ -11,7 +11,8 @@ from tasks.build_tags import filter_incompatible_tags, get_build_tags, get_default_build_tags from tasks.flavor import AgentFlavor -from tasks.libs.common.utils import REPO_PATH, TestsNotSupportedError, bin_name, get_build_flags, get_root +from tasks.gointegrationtest import DOGSTATSD_IT_CONF, containerized_integration_tests +from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags, get_root from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars # constants @@ -174,29 +175,14 @@ def integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", t """ Run integration tests for dogstatsd """ - if sys.platform == 'win32': - raise TestsNotSupportedError('DogStatsD integration tests are not supported on Windows') - - go_build_tags = " ".join(get_default_build_tags(build="test")) - race_opt = "-race" if race else "" - exec_opts = "" - timeout_opt = f"-timeout {timeout}" if timeout else "" - - # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout - # to the call, we don't want them because while calling invoke below, invoke - # thinks that the parameters are for it to interpret. - # we're calling an intermediate script which only pass the binary name to the invoke task. - if remote_docker: - exec_opts = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" - - go_cmd = f'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}' - - prefixes = [ - "./test/integration/dogstatsd/...", - ] - - for prefix in prefixes: - ctx.run(f"{go_cmd} {prefix}") + containerized_integration_tests( + ctx, + DOGSTATSD_IT_CONF, + race=race, + remote_docker=remote_docker, + go_mod=go_mod, + timeout=timeout, + ) @task diff --git a/tasks/github_tasks.py b/tasks/github_tasks.py index 7cc67757a61c3..bf2c546c8ecb4 100644 --- a/tasks/github_tasks.py +++ b/tasks/github_tasks.py @@ -22,7 +22,6 @@ from tasks.libs.common.constants import DEFAULT_INTEGRATIONS_CORE_BRANCH from tasks.libs.common.datadog_api import create_gauge, send_event, send_metrics from tasks.libs.common.git import get_default_branch -from tasks.libs.common.junit_upload_core import repack_macos_junit_tar from tasks.libs.common.utils import get_git_pretty_ref from tasks.libs.owners.linter import codeowner_has_orphans, directory_has_packages_without_owner from tasks.libs.owners.parsing import read_owners @@ -76,8 +75,6 @@ def trigger_macos( version_cache=None, retry_download=3, retry_interval=10, - fast_tests=None, - test_washer=False, integrations_core_ref=DEFAULT_INTEGRATIONS_CORE_BRANCH, ): """ @@ -107,26 +104,8 @@ def trigger_macos( version_cache_file_content=version_cache, integrations_core_ref=integrations_core_ref, ) - elif workflow_type == "test": - conclusion = _trigger_macos_workflow( - release_version, - destination, - retry_download, - retry_interval, - workflow_name="test.yaml", - datadog_agent_ref=datadog_agent_ref, - version_cache_file_content=version_cache, - fast_tests=fast_tests, - test_washer=test_washer, - ) - repack_macos_junit_tar(conclusion, "junit-tests_macos.tgz", "junit-tests_macos-repacked.tgz") - elif workflow_type == "lint": - conclusion = _trigger_macos_workflow( - release_version, - workflow_name="lint.yaml", - datadog_agent_ref=datadog_agent_ref, - version_cache_file_content=version_cache, - ) + else: + raise Exit(f"Unsupported workflow type: {workflow_type}", code=1) if conclusion != "success": raise Exit(message=f"Macos {workflow_type} workflow {conclusion}", code=1) diff --git a/tasks/go.py b/tasks/go.py index 13f451cce050f..5bb3fcad7d656 100644 --- a/tasks/go.py +++ b/tasks/go.py @@ -78,11 +78,16 @@ def lint_module(target): concurrency_arg = "" if concurrency is None else f"--concurrency {concurrency}" tags_arg = " ".join(sorted(set(tags))) timeout_arg_value = "25m0s" if not timeout else f"{timeout}m0s" - return ctx.run( + res = ctx.run( f'golangci-lint run {verbosity} --timeout {timeout_arg_value} {concurrency_arg} --build-tags "{tags_arg}" --path-prefix "{module_path}" {golangci_lint_kwargs} {target}/...', env=env, warn=True, ) + # early stop on SIGINT: exit code is 128 + signal number, SIGINT is 2, so 130 + # for some reason this becomes -2 here + if res is not None and (res.exited == -2 or res.exited == 130): + raise KeyboardInterrupt() + return res target_path = Path(module_path) / target result, time_result = TimedOperationResult.run( diff --git a/tasks/gointegrationtest.py b/tasks/gointegrationtest.py new file mode 100644 index 0000000000000..1d5cc56c8b9cf --- /dev/null +++ b/tasks/gointegrationtest.py @@ -0,0 +1,161 @@ +import os +import sys +import traceback +from dataclasses import dataclass + +from invoke import Context, task +from invoke.exceptions import Exit + +from tasks.build_tags import get_default_build_tags +from tasks.libs.common.utils import TestsNotSupportedError, gitlab_section + + +@dataclass +class IntegrationTest: + """ + Integration tests + """ + + prefix: str + dir: str = None + extra_args: str = None + + +@dataclass +class IntegrationTestsConfig: + """ + Integration tests configuration + """ + + name: str + go_build_tags: list[str] + tests: list[IntegrationTest] + env: dict[str, str] = None + is_windows_supported: bool = True + + +CORE_AGENT_LINUX_IT_CONF = IntegrationTestsConfig( + name="Core Agent Linux", + go_build_tags=get_default_build_tags(build="test"), + tests=[ + IntegrationTest(prefix="./test/integration/config_providers/..."), + IntegrationTest(prefix="./test/integration/corechecks/..."), + IntegrationTest(prefix="./test/integration/listeners/..."), + IntegrationTest(prefix="./test/integration/util/kubelet/..."), + ], + is_windows_supported=False, +) + +CORE_AGENT_WINDOWS_IT_CONF = IntegrationTestsConfig( + name="Core Agent Windows", + go_build_tags=get_default_build_tags(build="test"), + tests=[ + # Run eventlog tests with the Windows API, which depend on the EventLog service + IntegrationTest(dir="./pkg/util/winutil/", prefix="./eventlog/...", extra_args="-evtapi Windows"), + # Run eventlog tailer tests with the Windows API, which depend on the EventLog service + IntegrationTest(dir=".", prefix="./pkg/logs/tailers/windowsevent/...", extra_args="-evtapi Windows"), + # Run eventlog check tests with the Windows API, which depend on the EventLog service + # Don't include submodules, since the `-evtapi` flag is not defined in them + IntegrationTest( + dir=".", prefix="./comp/checks/windowseventlog/windowseventlogimpl/check", extra_args="-evtapi Windows" + ), + ], +) + +DOGSTATSD_IT_CONF = IntegrationTestsConfig( + name="DogStatsD", + go_build_tags=get_default_build_tags(build="test"), + tests=[IntegrationTest(prefix="./test/integration/dogstatsd/...")], + is_windows_supported=False, +) + +CLUSTER_AGENT_IT_CONF = IntegrationTestsConfig( + name="Cluster Agent", + go_build_tags=get_default_build_tags(build="cluster-agent") + ["docker", "test"], + tests=[ + IntegrationTest(prefix="./test/integration/util/kube_apiserver"), + IntegrationTest(prefix="./test/integration/util/leaderelection"), + ], + is_windows_supported=False, +) + +TRACE_AGENT_IT_CONF = IntegrationTestsConfig( + name="Trace Agent", + go_build_tags=get_default_build_tags(build="test"), + tests=[IntegrationTest(prefix="./cmd/trace-agent/test/testsuite/...")], + env={"INTEGRATION": "yes"}, + is_windows_supported=False, +) + + +def containerized_integration_tests( + ctx: Context, + integration_tests_config: IntegrationTestsConfig, + race=False, + remote_docker=False, + go_mod="readonly", + timeout="", +): + if sys.platform == 'win32' and not integration_tests_config.is_windows_supported: + raise TestsNotSupportedError(f'{integration_tests_config.name} integration tests are not supported on Windows') + test_args = { + "go_mod": go_mod, + "go_build_tags": " ".join(integration_tests_config.go_build_tags), + "race_opt": "-race" if race else "", + "exec_opts": "", + "timeout_opt": f"-timeout {timeout}" if timeout else "", + } + + # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout + # to the call, we don't want them because while calling invoke below, invoke + # thinks that the parameters are for it to interpret. + # we're calling an intermediate script which only pass the binary name to the invoke task. + if remote_docker: + test_args["exec_opts"] = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" + + go_cmd = 'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) # noqa: FS002 + + for it in integration_tests_config.tests: + if it.dir: + with ctx.cd(f"{it.dir}"): + ctx.run(f"{go_cmd} {it.prefix}", env=integration_tests_config.env) + else: + ctx.run(f"{go_cmd} {it.prefix}", env=integration_tests_config.env) + + +@task +def integration_tests(ctx, race=False, remote_docker=False, timeout=""): + """ + Run all the available integration tests + """ + core_agent_conf = CORE_AGENT_WINDOWS_IT_CONF if sys.platform == 'win32' else CORE_AGENT_LINUX_IT_CONF + tests = { + "Agent Core": lambda: containerized_integration_tests( + ctx, core_agent_conf, race=race, remote_docker=remote_docker, timeout=timeout + ), + "DogStatsD": lambda: containerized_integration_tests( + ctx, DOGSTATSD_IT_CONF, race=race, remote_docker=remote_docker, timeout=timeout + ), + "Cluster Agent": lambda: containerized_integration_tests( + ctx, CLUSTER_AGENT_IT_CONF, race=race, remote_docker=remote_docker, timeout=timeout + ), + "Trace Agent": lambda: containerized_integration_tests(ctx, TRACE_AGENT_IT_CONF, race=race, timeout=timeout), + } + tests_failures = {} + for t_name, t in tests.items(): + with gitlab_section(f"Running the {t_name} integration tests", collapsed=True, echo=True): + try: + t() + except TestsNotSupportedError as e: + print(f"Skipping {t_name}: {e}") + except Exception: + # Keep printing the traceback not to have to wait until all tests are done to see what failed + traceback.print_exc() + # Storing the traceback to print it at the end without directly raising the exception + tests_failures[t_name] = traceback.format_exc() + if tests_failures: + print("The following integration tests failed:") + for t_name in tests_failures: + print(f"- {t_name}") + print("See the above logs to get the full traceback.") + raise Exit(code=1) diff --git a/tasks/gotest.py b/tasks/gotest.py index 7ca411e7521ea..af889e8b6dcae 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -11,7 +11,6 @@ import os import re import sys -import traceback from collections import defaultdict from collections.abc import Iterable from datetime import datetime @@ -22,13 +21,10 @@ from invoke.context import Context from invoke.exceptions import Exit -from tasks.agent import integration_tests as agent_integration_tests from tasks.build_tags import compute_build_tags_for_flavor -from tasks.cluster_agent import integration_tests as dca_integration_tests from tasks.collector import OCB_VERSION from tasks.coverage import PROFILE_COV, CodecovWorkaround from tasks.devcontainer import run_on_devcontainer -from tasks.dogstatsd import integration_tests as dsd_integration_tests from tasks.flavor import AgentFlavor from tasks.libs.common.color import color_message from tasks.libs.common.datadog_api import create_count, send_metrics @@ -36,7 +32,6 @@ from tasks.libs.common.gomodules import get_default_modules from tasks.libs.common.junit_upload_core import enrich_junitxml, produce_junit_tar from tasks.libs.common.utils import ( - TestsNotSupportedError, clean_nested_paths, get_build_flags, gitlab_section, @@ -46,7 +41,6 @@ from tasks.modules import GoModule, get_module_by_path from tasks.test_core import ModuleTestResult, process_input_args, process_module_results, test_core from tasks.testwasher import TestWasher -from tasks.trace_agent import integration_tests as trace_integration_tests from tasks.update_go import PATTERN_MAJOR_MINOR_BUGFIX GO_TEST_RESULT_TMP_JSON = 'module_test_output.json' @@ -151,6 +145,9 @@ def command(test_results, module, module_result): out_stream=test_profiler, warn=True, ) + # early stop on SIGINT: exit code is 128 + signal number, SIGINT is 2, so 130 + if res is not None and res.exited == 130: + raise KeyboardInterrupt() module_result.result_json_path = os.path.join(module_path, GO_TEST_RESULT_TMP_JSON) @@ -405,36 +402,6 @@ def test( print(f"Tests final status (including re-runs): {color_message('ALL TESTS PASSED', 'green')}") -@task -def integration_tests(ctx, race=False, remote_docker=False, timeout=""): - """ - Run all the available integration tests - """ - tests = { - "Agent": lambda: agent_integration_tests(ctx, race=race, remote_docker=remote_docker, timeout=timeout), - "DogStatsD": lambda: dsd_integration_tests(ctx, race=race, remote_docker=remote_docker, timeout=timeout), - "Cluster Agent": lambda: dca_integration_tests(ctx, race=race, remote_docker=remote_docker, timeout=timeout), - "Trace Agent": lambda: trace_integration_tests(ctx, race=race, timeout=timeout), - } - tests_failures = {} - for t_name, t in tests.items(): - with gitlab_section(f"Running the {t_name} integration tests", collapsed=True, echo=True): - try: - t() - except TestsNotSupportedError as e: - print(f"Skipping {t_name}: {e}") - except Exception: - # Keep printing the traceback not to have to wait until all tests are done to see what failed - traceback.print_exc() - # Storing the traceback to print it at the end without directly raising the exception - tests_failures[t_name] = traceback.format_exc() - if tests_failures: - print("Integration tests failed:") - for t_name, t_failure in tests_failures.items(): - print(f"{t_name}:\n{t_failure}") - raise Exit(code=1) - - @task def e2e_tests(ctx, target="gitlab", agent_image="", dca_image="", argo_workflow="default"): """ diff --git a/tasks/kmt.py b/tasks/kmt.py index 18f5f9370893b..b284e6d56c91b 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -738,17 +738,18 @@ def prepare( ) if ci: - domains = None stack = "ci" return _prepare(ctx, stack, component, arch_obj, packages, verbose, ci, compile_only) - if alien_vms is not None: - err_msg = f"no alient VMs discovered from provided profile {alien_vms}." - else: - err_msg = f"no vms found from list {vms}. Run `inv -e kmt.status` to see all VMs in current stack" - stack = get_kmt_or_alien_stack(ctx, stack, vms, alien_vms) - domains = get_target_domains(ctx, stack, ssh_key, arch_obj, vms, alien_vms) - assert len(domains) > 0, err_msg + domains = None + if not compile_only: + if alien_vms is not None: + err_msg = f"no alient VMs discovered from provided profile {alien_vms}." + else: + err_msg = f"no vms found from list {vms}. Run `inv -e kmt.status` to see all VMs in current stack" + stack = get_kmt_or_alien_stack(ctx, stack, vms, alien_vms) + domains = get_target_domains(ctx, stack, ssh_key, arch_obj, vms, alien_vms) + assert len(domains) > 0, err_msg _prepare(ctx, stack, component, arch_obj, packages, verbose, ci, compile_only, domains=domains) diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index a32546d23ee38..54a3ba49a43f1 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -332,7 +332,9 @@ def workflow_run_for_ref_after_date(self, workflow_name, ref, oldest_date): return sorted(recent_runs, key=lambda run: run.created_at, reverse=True) - def latest_release(self) -> str: + def latest_release(self, major_version=7) -> str: + if major_version == 6: + return max((r for r in self.get_releases() if r.title.startswith('6.53')), key=lambda r: r.created_at).title release = self._repository.get_latest_release() return release.title @@ -523,6 +525,9 @@ def create_label(self, name, color, description=""): """ return self._repository.create_label(name, color, description) + def create_milestone(self, title): + self._repository.create_milestone(title) + def create_release(self, tag, message, draft=True): return self._repository.create_git_release( tag=tag, diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index cb4e42644e7c6..f1c0b4784d2f6 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -1258,19 +1258,26 @@ def full_config_get_all_stages(full_config: dict) -> set[str]: return all_stages -def update_test_infra_def(file_path, image_tag): +def update_test_infra_def(file_path, image_tag, is_dev_image=False, prefix_comment=""): """ - Override TEST_INFRA_DEFINITIONS_BUILDIMAGES in `.gitlab/common/test_infra_version.yml` file + Updates TEST_INFRA_DEFINITIONS_BUILDIMAGES in `.gitlab/common/test_infra_version.yml` file """ - with open(file_path) as gl: - file_content = gl.readlines() - with open(file_path, "w") as gl: - for line in file_content: - test_infra_def = re.search(r"TEST_INFRA_DEFINITIONS_BUILDIMAGES:\s*(\w+)", line) - if test_infra_def: - gl.write(line.replace(test_infra_def.group(1), image_tag)) + test_infra_def = {} + with open(file_path) as test_infra_version_file: + try: + test_infra_def = yaml.safe_load(test_infra_version_file) + test_infra_def["variables"]["TEST_INFRA_DEFINITIONS_BUILDIMAGES"] = image_tag + if is_dev_image: + test_infra_def["variables"]["TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX"] = "-dev" else: - gl.write(line) + test_infra_def["variables"]["TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX"] = "" + except yaml.YAMLError as e: + raise Exit(f"Error while loading {file_path}: {e}") from e + with open(file_path, "w") as test_infra_version_file: + test_infra_version_file.write(prefix_comment + ('\n\n' if prefix_comment else '')) + # Add explicit_start=True to keep the document start marker --- + # See "Document Start" in https://www.yaml.info/learn/document.html for more details + yaml.dump(test_infra_def, test_infra_version_file, explicit_start=True) def update_gitlab_config(file_path, tag, images="", test=True, update=True): diff --git a/tasks/libs/common/datadog_api.py b/tasks/libs/common/datadog_api.py index 55b888678b768..7b8b9df7f6a8a 100644 --- a/tasks/libs/common/datadog_api.py +++ b/tasks/libs/common/datadog_api.py @@ -16,8 +16,12 @@ def create_metric(metric_type, metric_name, timestamp, value, tags, unit=None, m unit = unit or unset metadata = unset - if metric_origin: - metadata = MetricMetadata(origin=MetricOrigin(**metric_origin)) + origin_metadata = metric_origin or { + "origin_product": 10, # Agent + "origin_sub_product": 54, # Agent CI + "origin_product_detail": 64, # Gitlab + } + metadata = MetricMetadata(origin=MetricOrigin(**origin_metadata)) return MetricSeries( metric=metric_name, diff --git a/tasks/libs/common/git.py b/tasks/libs/common/git.py index b50184c3af411..b1dc1b1cd7683 100644 --- a/tasks/libs/common/git.py +++ b/tasks/libs/common/git.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import subprocess import sys import tempfile from contextlib import contextmanager @@ -292,3 +293,20 @@ def get_last_release_tag(ctx, repo, pattern): last_tag_name = last_tag_name_with_suffix.removesuffix("^{}") last_tag_name = last_tag_name.removeprefix("refs/tags/") return last_tag_commit, last_tag_name + + +def get_git_config(key): + result = subprocess.run(['git', 'config', '--get', key], capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else None + + +def set_git_config(key, value): + subprocess.run(['git', 'config', key, value]) + + +def revert_git_config(original_config): + for key, value in original_config.items(): + if value is None: + subprocess.run(['git', 'config', '--unset', key]) + else: + subprocess.run(['git', 'config', key, value]) diff --git a/tasks/libs/common/junit_upload_core.py b/tasks/libs/common/junit_upload_core.py index b7282d9bf4ebc..cebbe0a1c2b75 100644 --- a/tasks/libs/common/junit_upload_core.py +++ b/tasks/libs/common/junit_upload_core.py @@ -377,25 +377,3 @@ def produce_junit_tar(files, result_path): job_env_info.size = job_env_file.getbuffer().nbytes job_env_file.seek(0) tgz.addfile(job_env_info, job_env_file) - - -def repack_macos_junit_tar(workflow_conclusion, infile, outfile): - """ - Repacks JUnit tgz file from macOS Github Action run, so it would - contain correct job name and job URL. - """ - if workflow_conclusion == "cancelled" and not os.path.exists(infile): - print(f"Skipping repacking of JUnit tarball due to {workflow_conclusion} workflow") - return - - with tarfile.open(infile) as infp, tarfile.open(outfile, "w:gz") as outfp, tempfile.TemporaryDirectory() as tempd: - infp.extractall(tempd) - - # write the proper job url and job name - with open(os.path.join(tempd, JOB_ENV_FILE_NAME), "w") as fp: - fp.write(f'CI_JOB_URL={os.environ.get("CI_JOB_URL", "")}\n') - fp.write(f'CI_JOB_NAME={os.environ.get("CI_JOB_NAME", "")}') - - # pack all files to a new tarball - for f in os.listdir(tempd): - outfp.add(os.path.join(tempd, f), arcname=f) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 38208c6a4d1fe..1887cffb97133 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -116,6 +116,7 @@ def env_filter(item): "GET_SOURCES_ATTEMPTS", "GO_TEST_SKIP_FLAKE", "GONOSUMDB", + "GOPROXY", "HELM_HOOKS_CI_IMAGE", "HELM_HOOKS_PERIODICAL_REBUILD_CONDUCTOR_ENV", "HOME", diff --git a/tasks/libs/package/size.py b/tasks/libs/package/size.py index 4743dfcd906de..2323d40fdb56b 100644 --- a/tasks/libs/package/size.py +++ b/tasks/libs/package/size.py @@ -37,22 +37,22 @@ # The below template contains the relative increase threshold for each package type PACKAGE_SIZE_TEMPLATE = { 'amd64': { - 'datadog-agent': {'deb': 140 * pow(10, 6)}, - 'datadog-iot-agent': {'deb': 10 * pow(10, 6)}, - 'datadog-dogstatsd': {'deb': 10 * pow(10, 6)}, - 'datadog-heroku-agent': {'deb': 70 * pow(10, 6)}, + 'datadog-agent': {'deb': int(140e6)}, + 'datadog-iot-agent': {'deb': int(10e6)}, + 'datadog-dogstatsd': {'deb': int(10e6)}, + 'datadog-heroku-agent': {'deb': int(70e6)}, }, 'x86_64': { - 'datadog-agent': {'rpm': 140 * pow(10, 6), 'suse': 140 * pow(10, 6)}, - 'datadog-iot-agent': {'rpm': 10 * pow(10, 6), 'suse': 10 * pow(10, 6)}, - 'datadog-dogstatsd': {'rpm': 10 * pow(10, 6), 'suse': 10 * pow(10, 6)}, + 'datadog-agent': {'rpm': int(140e6), 'suse': int(140e6)}, + 'datadog-iot-agent': {'rpm': int(10e6), 'suse': int(10e6)}, + 'datadog-dogstatsd': {'rpm': int(10e6), 'suse': int(10e6)}, }, 'arm64': { - 'datadog-agent': {'deb': 140 * pow(10, 6)}, - 'datadog-iot-agent': {'deb': 10 * pow(10, 6)}, - 'datadog-dogstatsd': {'deb': 10 * pow(10, 6)}, + 'datadog-agent': {'deb': int(140e6)}, + 'datadog-iot-agent': {'deb': int(10e6)}, + 'datadog-dogstatsd': {'deb': int(10e6)}, }, - 'aarch64': {'datadog-agent': {'rpm': 140 * pow(10, 6)}, 'datadog-iot-agent': {'rpm': 10 * pow(10, 6)}}, + 'aarch64': {'datadog-agent': {'rpm': int(140e6)}, 'datadog-iot-agent': {'rpm': int(10e6)}}, } @@ -177,25 +177,11 @@ def compare(ctx, package_sizes, ancestor, pkg_size): return pkg_size -def mb(value): - return f"{value / 1000000:.2f}MB" - - def _get_uncompressed_size(ctx, package, os_name): if os_name == 'deb': - return _get_deb_uncompressed_size(ctx, package) + return ( + int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs', hide=True).stdout) + * 1024 + ) else: - return _get_rpm_uncompressed_size(ctx, package) - - -def _get_deb_uncompressed_size(ctx, package): - # the size returned by dpkg is a number of bytes divided by 1024 - # so we multiply it back to get the same unit as RPM or stat - return ( - int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs', hide=True).stdout) - * 1024 - ) - - -def _get_rpm_uncompressed_size(ctx, package): - return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs', hide=True).stdout) + return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs', hide=True).stdout) diff --git a/tasks/libs/package/utils.py b/tasks/libs/package/utils.py index ec8363fcfa134..8f43eaa9b4e33 100644 --- a/tasks/libs/package/utils.py +++ b/tasks/libs/package/utils.py @@ -20,6 +20,7 @@ def __init__(self, arch, flavor, os_name, threshold): self.size = 0 self.ancestor_size = 0 self.diff = 0 + self.mb_diff = 0 self.threshold = threshold self.emoji = "✅" @@ -48,14 +49,15 @@ def compare(self, size, ancestor_size): self.size = size self.ancestor_size = ancestor_size self.diff = self.size - self.ancestor_size + self.mb_diff = float(f"{self.diff / pow(10, 6):.2f}") if self.ko(): self.emoji = "❌" - elif self.diff > 0: + elif self.mb_diff > 0: self.emoji = "⚠️" @staticmethod def mb(value): - return f"{value / 1000000:.2f}MB" + return f"{value / 1e6:.2f}MB" def log(self): return f"{self.emoji} - {self.name} size {self.mb(self.size)}: {self.mb(self.diff)} diff[{self.diff}] with previous {self.mb(self.ancestor_size)} (max: {self.mb(self.threshold)})" diff --git a/tasks/libs/releasing/documentation.py b/tasks/libs/releasing/documentation.py index 13c7f0d148252..9834983792cbd 100644 --- a/tasks/libs/releasing/documentation.py +++ b/tasks/libs/releasing/documentation.py @@ -31,7 +31,7 @@ def release_entry_for(agent_major_version): return f"release-a{agent_major_version}" -def create_release_page(version, freeze_date): +def create_release_page(version, cutoff_date): username = os.environ['ATLASSIAN_USERNAME'] password = os.environ['ATLASSIAN_PASSWORD'] parent_page_id = "2244936127" @@ -44,7 +44,7 @@ def create_release_page(version, freeze_date): page = confluence.create_page( space=SPACE_KEY, title=page_title, - body=create_release_table(version, freeze_date, teams), + body=create_release_table(version, cutoff_date, teams), parent_id=parent_page_id, editor="v2", ) @@ -52,7 +52,7 @@ def create_release_page(version, freeze_date): confluence.create_page( space=SPACE_KEY, title=f"{page_title} Notes", - body=create_release_notes(freeze_date, teams), + body=create_release_notes(cutoff_date, teams), parent_id=release_page["id"], ) return release_page @@ -118,12 +118,13 @@ def get_releasing_teams(): 'agent-platform', 'agent-release-management', 'container-ecosystems', + 'apm-trace-storage', } owners = set(list_owners()) return sorted(owners - non_releasing_teams) -def create_release_table(version, freeze_date, teams): +def create_release_table(version, cutoff_date, teams): from yattag import Doc doc, tag, text, line = Doc().ttl() @@ -153,7 +154,7 @@ def create_release_table(version, freeze_date, teams): with ( tag('td', colspan="2"), tag('p', style="text-align: center;"), - tag('time', datetime=f"{freeze_date + timedelta(days=26)}"), + tag('time', datetime=f"{cutoff_date + timedelta(days=26)}"), ): pass with tag('tr'): @@ -164,11 +165,11 @@ def create_release_table(version, freeze_date, teams): text(f'https://github.com/DataDog/datadog-agent/releases/tag/{version}') with tag('tr'): with tag('td'), tag('p'): - text('Code freeze date') + text('Cut-off date') with ( tag('td', colspan="2"), tag('p', style="text-align: center;"), - tag('time', datetime=f"{freeze_date}"), + tag('time', datetime=f"{cutoff_date}"), ): pass with tag('tr'): @@ -201,17 +202,17 @@ def create_release_table(version, freeze_date, teams): return doc.getvalue() -def create_release_notes(freeze_date, teams): +def create_release_notes(cutoff_date, teams): from yattag import Doc doc, tag, text, line = Doc().ttl() milestones = { - '"Code freeze"': freeze_date, - '"RC.1 built"': freeze_date + timedelta(days=1), - '"Staging deployment"': freeze_date + timedelta(days=3), - '"Prod deployment start"': freeze_date + timedelta(days=11), - '"Full prod deployment"': freeze_date + timedelta(days=20), - '"Release"': freeze_date + timedelta(days=26), + '"Cut-off"': cutoff_date, + '"RC.1 built"': cutoff_date + timedelta(days=1), + '"Staging deployment"': cutoff_date + timedelta(days=3), + '"Prod deployment start"': cutoff_date + timedelta(days=11), + '"Full prod deployment"': cutoff_date + timedelta(days=20), + '"Release"': cutoff_date + timedelta(days=26), } line('h2', 'Schedule') diff --git a/tasks/libs/releasing/version.py b/tasks/libs/releasing/version.py index 23472d57c0cca..85e66ea283080 100644 --- a/tasks/libs/releasing/version.py +++ b/tasks/libs/releasing/version.py @@ -6,6 +6,7 @@ from invoke import Exit from tasks.libs.ciproviders.github_api import GithubAPI +from tasks.libs.common.color import Color, color_message from tasks.libs.common.constants import ( AGENT_VERSION_CACHE_NAME, ALLOWED_REPO_NIGHTLY_BRANCHES, @@ -441,3 +442,80 @@ def get_matching_pattern(ctx, major_version, release=False): ) pattern = max(tags, key=cmp_to_key(semver.compare)) return pattern + + +def deduce_version(ctx, branch, as_str: bool = True, trust: bool = False, next_version: bool = True) -> str | Version: + """Deduces the version from the release branch name. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + """ + release_version = get_next_version_from_branch(ctx, branch, as_str=as_str, next_version=next_version) + + print( + f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr + ) + + if ( + trust + or not os.isatty(sys.stdin.fileno()) + or yes_no_question( + 'Is this the version you want to use?', + color="orange", + default=False, + ) + ): + return release_version + + raise Exit(color_message("Aborting.", "red"), code=1) + + +def get_version_major(branch: str) -> int: + """Get the major version from a branch name.""" + + return 7 if branch == 'main' else int(branch.split('.')[0]) + + +def get_all_version_tags(ctx) -> list[str]: + """Returns the tags for all the versions of the Agent in git.""" + + cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" + + return ctx.run(cmd, hide=True).stdout.strip().split('\n') + + +def get_next_version_from_branch(ctx, branch: str, as_str: bool = True, next_version: bool = True) -> str | Version: + """Returns the latest version + 1 belonging to a branch. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + + Example: + get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. + get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. + """ + + re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") + + try: + matched = re_branch.match(branch).group(1) + except Exception as e: + raise Exit( + f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 + ) from e + + tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] + versions = sorted(Version(*tag) for tag in tags) + + minor, major = tuple(map(int, branch.split('.')[:2])) + + if next_version: + # Get version after the latest one + version = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) + else: + # Get current latest version + assert versions, f"No tags found for branch {branch} (expected at least one tag)" + + version = versions[-1] + + return str(version) if as_str else version diff --git a/tasks/package.py b/tasks/package.py index f2d49130152dc..3a831a24225fc 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -8,15 +8,12 @@ from tasks.libs.common.git import get_default_branch from tasks.libs.package.size import ( PACKAGE_SIZE_TEMPLATE, - _get_deb_uncompressed_size, - _get_rpm_uncompressed_size, compare, compute_package_size_metrics, ) from tasks.libs.package.utils import ( PackageSize, display_message, - find_package, get_ancestor, list_packages, retrieve_package_sizes, @@ -61,47 +58,6 @@ def check_size(ctx, filename: str = 'package_sizes.json', dry_run: bool = False) raise Exit(code=1) -@task -def compare_size(ctx, new_package, stable_package, package_type, last_stable, threshold): - mb = 1000000 - - if package_type.endswith('deb'): - new_package_size = _get_deb_uncompressed_size(ctx, find_package(new_package)) - stable_package_size = _get_deb_uncompressed_size(ctx, find_package(stable_package)) - else: - new_package_size = _get_rpm_uncompressed_size(ctx, find_package(new_package)) - stable_package_size = _get_rpm_uncompressed_size(ctx, find_package(stable_package)) - - threshold = int(threshold) - - diff = new_package_size - stable_package_size - - # For printing purposes - new_package_size_mb = new_package_size / mb - stable_package_size_mb = stable_package_size / mb - threshold_mb = threshold / mb - diff_mb = diff / mb - - if diff > threshold: - print( - color_message( - f"""{package_type} size increase is too large: - New package size is {new_package_size_mb:.2f}MB - Stable package ({last_stable}) size is {stable_package_size_mb:.2f}MB - Diff is {diff_mb:.2f}MB > {threshold_mb:.2f}MB (max allowed diff)""", - "red", - ) - ) - raise Exit(code=1) - - print( - f"""{package_type} size increase is OK: - New package size is {new_package_size_mb:.2f}MB - Stable package ({last_stable}) size is {stable_package_size_mb:.2f}MB - Diff is {diff_mb:.2f}MB (max allowed diff: {threshold_mb:.2f}MB)""" - ) - - @task def send_size( ctx, diff --git a/tasks/pipeline.py b/tasks/pipeline.py index 59aa6a3666172..ad825bee1540d 100644 --- a/tasks/pipeline.py +++ b/tasks/pipeline.py @@ -131,6 +131,9 @@ def auto_cancel_previous_pipelines(ctx): raise Exit("GITLAB_TOKEN variable needed to cancel pipelines on the same ref.", 1) git_ref = os.environ["CI_COMMIT_REF_NAME"] + if git_ref == "": + raise Exit("CI_COMMIT_REF_NAME is empty, skipping pipeline cancellation", 0) + git_sha = os.getenv("CI_COMMIT_SHA") repo = get_gitlab_repo() diff --git a/tasks/release.py b/tasks/release.py index aa16b7fd7045f..39274fc027bc1 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -34,6 +34,7 @@ get_last_commit, get_last_release_tag, is_agent6, + set_git_config, try_git_command, ) from tasks.libs.common.gomodules import get_default_modules @@ -69,10 +70,11 @@ VERSION_RE, _create_version_from_match, current_version, + deduce_version, + get_version_major, next_final_version, next_rc_version, ) -from tasks.libs.types.version import Version from tasks.pipeline import edit_schedule, run from tasks.release_metrics.metrics import get_prs_metrics, get_release_lead_time @@ -84,68 +86,6 @@ BACKPORT_LABEL_COLOR = "5319e7" -def deduce_version(ctx, branch, as_str=True, trust=False) -> str | Version: - release_version = get_next_version_from_branch(ctx, branch, as_str=as_str) - - print( - f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr - ) - - if ( - trust - or not os.isatty(sys.stdin.fileno()) - or yes_no_question( - 'Is this the version you want to use?', - color="orange", - default=False, - ) - ): - return release_version - - raise Exit(color_message("Aborting.", "red"), code=1) - - -def get_version_major(branch: str) -> int: - """Get the major version from a branch name.""" - - return 7 if branch == 'main' else int(branch.split('.')[0]) - - -def get_all_version_tags(ctx) -> list[str]: - """Returns the tags for all the versions of the Agent in git.""" - - cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" - - return ctx.run(cmd, hide=True).stdout.strip().split('\n') - - -def get_next_version_from_branch(ctx, branch: str, as_str=True) -> str | Version: - """Returns the latest version + 1 belonging to a branch. - - Example: - get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. - get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. - """ - - re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") - - try: - matched = re_branch.match(branch).group(1) - except Exception as e: - raise Exit( - f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 - ) from e - - tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] - versions = sorted(Version(*tag) for tag in tags) - - minor, major = tuple(map(int, branch.split('.')[:2])) - - latest = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) - - return str(latest) if as_str else latest - - @task def list_major_change(_, milestone): """List all PR labeled "major_changed" for this release.""" @@ -432,11 +372,16 @@ def create_rc(ctx, release_branch, patch_version=False, upstream="origin", slack This also requires that there are no local uncommitted changes, that the current branch is 'main' or the release branch, and that no branch named 'release/' already exists locally or upstream. """ - major_version = get_version_major(release_branch) with agent_context(ctx, release_branch): github = GithubAPI(repository=GITHUB_REPO_NAME) + github_action = os.environ.get("GITHUB_ACTIONS") + + if github_action: + set_git_config('user.name', 'github-actions[bot]') + set_git_config('user.email', 'github-actions[bot]@users.noreply.github.com') + upstream = f"https://x-access-token:{os.environ.get('GITHUB_TOKEN')}@github.com/{GITHUB_REPO_NAME}.git" # Get the version of the highest major: useful for some logging & to get # the version to use for Go submodules updates @@ -491,7 +436,9 @@ def create_rc(ctx, release_branch, patch_version=False, upstream="origin", slack ctx.run("git ls-files . | grep 'go.mod$' | xargs git add") ok = try_git_command( - ctx, f"git commit --no-verify -m 'Update release.json and Go modules for {new_highest_version}'" + ctx, + f"git commit --no-verify -m 'Update release.json and Go modules for {new_highest_version}'", + github_action, ) if not ok: raise Exit( @@ -723,10 +670,8 @@ def create_release_branches(ctx, base_directory="~/dd", major_version: int = 7, github = GithubAPI(repository=GITHUB_REPO_NAME) current = current_version(ctx, major_version) - next = current.next_version(bump_minor=True) current.rc = False current.devel = False - next.devel = False # Strings with proper branch/tag names release_branch = current.branch() @@ -764,43 +709,7 @@ def create_release_branches(ctx, base_directory="~/dd", major_version: int = 7, ) # Step 2 - Create PRs with new settings in datadog-agent repository - # Step 2.0 - Create milestone update - milestone_branch = f"release_milestone-{int(time.time())}" - ctx.run(f"git switch -c {milestone_branch}") - rj = load_release_json() - rj["current_milestone"] = f"{next}" - _save_release_json(rj) - # Commit release.json - ctx.run("git add release.json") - ok = try_git_command(ctx, f"git commit -m 'Update release.json with current milestone to {next}'") - - if not ok: - raise Exit( - color_message( - f"Could not create commit. Please commit manually and push the commit to the {milestone_branch} branch.", - "red", - ), - code=1, - ) - - res = ctx.run(f"git push --set-upstream {upstream} {milestone_branch}", warn=True) - if res.exited is None or res.exited > 0: - raise Exit( - color_message( - f"Could not push branch {milestone_branch} to the upstream '{upstream}'. Please push it manually and then open a PR against {release_branch}.", - "red", - ), - code=1, - ) - - create_release_pr( - f"[release] Update current milestone to {next}", - get_default_branch(), - milestone_branch, - next, - ) - - # Step 2.1 - Update release.json + # Step 2.0 - Update release.json update_branch = f"{release_branch}-updates" ctx.run(f"git checkout {release_branch}") @@ -867,10 +776,6 @@ def _update_last_stable(_, version, major_version: int = 7): return release_json["current_milestone"] -def _get_agent6_latest_release(gh): - return max((r for r in gh.get_releases() if r.title.startswith('6.53')), key=lambda r: r.created_at).title - - @task def cleanup(ctx, release_branch): """Perform the post release cleanup steps @@ -880,13 +785,13 @@ def cleanup(ctx, release_branch): - Updates the release.json last_stable fields """ - with agent_context(ctx, release_branch): + # This task will create a PR to update the last_stable field in release.json + # It must create the PR against the default branch (6 or 7), so setting the context on it + main_branch = get_default_branch() + with agent_context(ctx, main_branch): gh = GithubAPI() major_version = get_version_major(release_branch) - if major_version == 6: - latest_release = _get_agent6_latest_release(gh) - else: - latest_release = gh.latest_release() + latest_release = gh.latest_release(major_version) match = VERSION_RE.search(latest_release) if not match: raise Exit(f'Unexpected version fetched from github {latest_release}', code=1) @@ -895,7 +800,6 @@ def cleanup(ctx, release_branch): current_milestone = _update_last_stable(ctx, version, major_version=major_version) # create pull request to update last stable version - main_branch = get_default_branch() cleanup_branch = f"release/{version}-cleanup" ctx.run(f"git checkout -b {cleanup_branch}") ctx.run("git add release.json") @@ -1064,7 +968,7 @@ def get_active_release_branch(ctx, release_branch): with agent_context(ctx, branch=release_branch): gh = GithubAPI() - next_version = get_next_version(gh, latest_release=_get_agent6_latest_release(gh) if is_agent6(ctx) else None) + next_version = get_next_version(gh, latest_release=gh.latest_release(6) if is_agent6(ctx) else None) release_branch = gh.get_branch(next_version.branch()) if release_branch: print(f"{release_branch.name}") @@ -1125,12 +1029,13 @@ def generate_release_metrics(ctx, milestone, freeze_date, release_date): print(code_stats) +# TODO rename to freeze_date to cutoff_date @task def create_schedule(_, version, freeze_date): """Create confluence pages for the release schedule. Args: - freeze_date: Date when the code freeze was started. Expected format YYYY-MM-DD, like '2022-02-01' + freeze_date: Date when the code cut-off happened. Expected format YYYY-MM-DD, like '2022-02-01' """ required_environment_variables = ["ATLASSIAN_USERNAME", "ATLASSIAN_PASSWORD"] @@ -1256,7 +1161,7 @@ def create_github_release(ctx, release_branch, draft=True): ) notes = [] - version = deduce_version(ctx, release_branch) + version = deduce_version(ctx, release_branch, next_version=False) with agent_context(ctx, release_branch): for section, filename in sections: @@ -1295,3 +1200,63 @@ def create_github_release(ctx, release_branch, draft=True): ) print(f"Link to the release note: {release.html_url}") + + +@task +def update_current_milestone(ctx, major_version: int = 7, upstream="origin"): + """ + Create a PR to bump the current_milestone in the release.json file + """ + import github + + gh = GithubAPI() + + current = current_version(ctx, major_version) + next = current.next_version(bump_minor=True) + next.devel = False + + print(f"Creating the {next} milestone...") + + try: + gh.create_milestone(str(next)) + except github.GithubException as e: + if e.status == 422: + print(f"Milestone {next} already exists") + else: + raise e + + with agent_context(ctx, get_default_branch(major=major_version)): + milestone_branch = f"release_milestone-{int(time.time())}" + ctx.run(f"git switch -c {milestone_branch}") + rj = load_release_json() + rj["current_milestone"] = f"{next}" + _save_release_json(rj) + # Commit release.json + ctx.run("git add release.json") + ok = try_git_command(ctx, f"git commit -m 'Update release.json with current milestone to {next}'") + + if not ok: + raise Exit( + color_message( + f"Could not create commit. Please commit manually and push the commit to the {milestone_branch} branch.", + Color.RED, + ), + code=1, + ) + + res = ctx.run(f"git push --set-upstream {upstream} {milestone_branch}", warn=True) + if res.exited is None or res.exited > 0: + raise Exit( + color_message( + f"Could not push branch {milestone_branch} to the upstream '{upstream}'. Please push it manually and then open a PR against main.", + Color.RED, + ), + code=1, + ) + + create_release_pr( + f"[release] Update current milestone to {next}", + get_default_branch(), + milestone_branch, + next, + ) diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 6f8e4e1f7457e..ddd731b535bea 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -2024,3 +2024,40 @@ def copy_ebpf_and_related_files(ctx: Context, target: Path | str, arch: Arch | N ctx.run(f"chmod 0444 {target}/*.o {target}/*.c {target}/co-re/*.o") ctx.run(f"cp /opt/datadog-agent/embedded/bin/clang-bpf {target}") ctx.run(f"cp /opt/datadog-agent/embedded/bin/llc-bpf {target}") + + +@task +def build_usm_debugger( + ctx, + arch: str = CURRENT_ARCH, + strip_binary=False, +): + build_object_files(ctx) + + build_dir = os.path.join("pkg", "ebpf", "bytecode", "build", arch) + + # copy compilation artifacts to the debugger root directory for the purposes of embedding + usm_programs = [ + os.path.join(build_dir, "co-re", "usm-debug.o"), + os.path.join(build_dir, "co-re", "shared-libraries-debug.o"), + ] + + embedded_dir = os.path.join(".", "pkg", "network", "usm", "debugger", "cmd") + + for p in usm_programs: + print(p) + shutil.copy(p, embedded_dir) + + arch_obj = Arch.from_str(arch) + ldflags, gcflags, env = get_build_flags(ctx, arch=arch_obj) + + cmd = 'go build -tags="linux_bpf,usm_debugger" -o bin/usm-debugger -ldflags="{ldflags}" ./pkg/network/usm/debugger/cmd/' + + if strip_binary: + ldflags += ' -s -w' + + args = { + "ldflags": ldflags, + } + + ctx.run(cmd.format(**args), env=env) diff --git a/tasks/trace_agent.py b/tasks/trace_agent.py index d65ae6861fcd8..0ec1dd1956aa8 100644 --- a/tasks/trace_agent.py +++ b/tasks/trace_agent.py @@ -5,7 +5,8 @@ from tasks.build_tags import add_fips_tags, filter_incompatible_tags, get_build_tags, get_default_build_tags from tasks.flavor import AgentFlavor -from tasks.libs.common.utils import REPO_PATH, TestsNotSupportedError, bin_name, get_build_flags +from tasks.gointegrationtest import TRACE_AGENT_IT_CONF, containerized_integration_tests +from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars BIN_PATH = os.path.join(".", "bin", "trace-agent") @@ -81,15 +82,14 @@ def integration_tests(ctx, race=False, go_mod="readonly", timeout="10m"): """ Run integration tests for trace agent """ - if sys.platform == 'win32': - raise TestsNotSupportedError('Trace Agent integration tests are not supported on Windows') - - go_build_tags = " ".join(get_default_build_tags(build="test")) - race_opt = "-race" if race else "" - timeout_opt = f"-timeout {timeout}" if timeout else "" - - go_cmd = f'go test {timeout_opt} -mod={go_mod} {race_opt} -v -tags "{go_build_tags}"' - ctx.run(f"{go_cmd} ./cmd/trace-agent/test/testsuite/...", env={"INTEGRATION": "yes"}) + containerized_integration_tests( + ctx, + TRACE_AGENT_IT_CONF, + race=race, + remote_docker=False, + go_mod=go_mod, + timeout=timeout, + ) @task diff --git a/tasks/unit_tests/package_lib_tests.py b/tasks/unit_tests/package_lib_tests.py index 984e4cc4f871c..72f3c6c190252 100644 --- a/tasks/unit_tests/package_lib_tests.py +++ b/tasks/unit_tests/package_lib_tests.py @@ -238,6 +238,26 @@ def test_on_branch_warning(self, mock_print): f"⚠️ - {flavor}-{arch}-{os_name} size 69.00MB: 1.00MB diff[1000000] with previous 68.00MB (max: 70.00MB)" ) + @patch.dict( + 'os.environ', + {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'pikachu'}, + ) + @patch('builtins.print') + def test_on_branch_ok_small_diff(self, mock_print): + flavor, arch, os_name = 'datadog-agent', 'aarch64', 'suse' + s = PackageSize(arch, flavor, os_name, 70000000) + c = MockContext( + run={ + 'git merge-base HEAD origin/main': Result('25'), + f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.rpm | grep Size | cut -d : -f 2 | xargs": Result(68004999), + } + ) + res = compare(c, self.package_sizes, '25', s) + self.assertEqual(res.markdown(), "|datadog-agent-aarch64-suse|0.00MB|✅|68.00MB|68.00MB|70.00MB|") + mock_print.assert_called_with( + f"✅ - {flavor}-{arch}-{os_name} size 68.00MB: 0.00MB diff[4999] with previous 68.00MB (max: 70.00MB)" + ) + @patch.dict( 'os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'pikachu'} ) diff --git a/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 b/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 index 991317602152f..41bf71748cc2b 100644 --- a/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 +++ b/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 @@ -2,11 +2,8 @@ .SYNOPSIS Generates a Chocolatey package for the Datadog Agent. -.PARAMETER installMethod -Specifies the installation method. Valid values are "offline" and "online". This parameter is mandatory. - .PARAMETER msiDirectory -Specifies the directory containing the MSI file that will be used to calculate the checksum. This parameter is mandatory when the installMethod is "online". +Specifies the directory containing the MSI file that will be used to calculate the checksum. .PARAMETER Flavor Specifies the flavor of the Datadog Agent. The default value is "datadog-agent". @@ -18,23 +15,18 @@ Overrides the Agent version when building packages locally for testing. Indicates whether to install dependencies. The default value is $true. .EXAMPLE -.\Generate-Chocolatey-Package.ps1 -installMethod online -Flavor datadog-agent -VersionOverride "7.62.0" -msiDirectory C:\mnt\omnibus\pkg\ +.\Generate-Chocolatey-Package.ps1 -Flavor datadog-agent -VersionOverride "7.62.0" -msiDirectory C:\mnt\omnibus\pkg Generates a chocolatey package for 7.62.0, requires the MSI file to be present in MSIDirectory. .EXAMPLE -$env:CI_PIPELINE_ID="50910739"; .\Generate-Chocolatey-Package.ps1 -installMethod online -Flavor datadog-agent -VersionOverride "7.62.0-devel.git.276.e59b1b3.pipeline.50910739" -msiDirectory C:\mnt\omnibus\pkg +$env:CI_PIPELINE_ID="50910739"; .\Generate-Chocolatey-Package.ps1 -Flavor datadog-agent -VersionOverride "7.62.0-devel.git.276.e59b1b3.pipeline.50910739" -msiDirectory C:\mnt\omnibus\pkg Generates a chocolatey package for PR/devel build 7.62.0-devel.git.276.e59b1b3.pipeline.50910739, requires the MSI file to be present in MSIDirectory. The generated chocolatey package requires the MSI be uploaded to the dd-agent-mstesting bucket. #> Param( [Parameter(Mandatory=$true)] - [ValidateSet("offline", "online")] - [String] - $installMethod, - - [Parameter(Mandatory=$false)] [String] $msiDirectory, @@ -69,9 +61,9 @@ if (![string]::IsNullOrEmpty($VersionOverride)) { } $copyright = "Datadog {0}" -f (Get-Date).Year -$releasePattern = "(\d+\.\d+\.\d+)" -$releaseCandidatePattern = "(\d+\.\d+\.\d+)-rc\.(\d+)" -$develPattern = "(\d+\.\d+\.\d+)-devel\.git\.\d+\.(.+)" +$releasePattern = "^(\d+\.\d+\.\d+)$" +$releaseCandidatePattern = "^(\d+\.\d+\.\d+)-rc\.(\d+)$" +$develPattern = "^(\d+\.\d+\.\d+)-devel\.git\.\d+\.(.+)" # Build the package in a temporary directory # Some of the build steps modify the package source, so we don't want to do this in the source directory @@ -87,21 +79,16 @@ try { # For historical reasons, use a different artifact name for the datadog-agent flavor # See agent-release-management for more details $artifactName = "ddagent-cli" - $packageSource = "$repoRoot\chocolatey\datadog-agent\$installMethod" - $nuspecFile = "datadog-agent-$installMethod.nuspec" } elseif ($Flavor -eq "datadog-fips-agent") { - if ($installMethod -eq "offline") { - Write-Error "Offline install method not supported for flavor $Flavor" - exit 1 - } $artifactName = "datadog-fips-agent" - $packageSource = "$repoRoot\chocolatey\datadog-fips-agent\online" - $nuspecFile = "datadog-fips-agent-online.nuspec" } else { Write-Error "Unknown flavor $Flavor" exit 1 } + $packageSource = "$repoRoot\chocolatey\$Flavor" + $nuspecFile = "$Flavor.nuspec" + # These files/directories are referenced in the nuspec file $licensePath = "tools\LICENSE.txt" $installScript = "tools\chocolateyinstall.ps1" @@ -118,24 +105,22 @@ try { $releaseNotes = "https://github.com/DataDog/datadog-agent/releases/tag/{0}-rc.{1}" -f $agentVersionMatches.Matches.Groups[1], $agentVersionMatches.Matches.Groups[2] $url = "https://s3.amazonaws.com/dd-agent-mstesting/builds/beta/$artifactName-$($agentVersionMatches.Matches.Groups[1])-rc.$($agentVersionMatches.Matches.Groups[2]).msi" } elseif ($rawAgentVersion -match $develPattern) { - if ($installMethod -eq "online") { - # For devel builds/branches, use the dd-agent-mstesting bucket URL - # This allows us to build and test the package in PRs, and locally - # by using the `-VersionOverride` param. - if ([string]::IsNullOrEmpty($env:CI_PIPELINE_ID)) { - Write-Error "CI_PIPELINE_ID is not set, aborting" - exit 1 - } else { - if ($rawAgentVersion -notmatch $env:CI_PIPELINE_ID) { - Write-Error "CI_PIPELINE_ID is not found in the agent version, aborting" -ErrorAction Continue - if ([string]::IsNullOrEmpty($env:BUCKET_BRANCH)) { - # inv agent.version requires BUCKET_BRANCH to be set when including pipeline in version - Write-Error "BUCKET_BRANCH is not set, if you are running this locally, set `$env:BUCKET_BRANCH='dev' or pass the -VersionOverride parameter" -ErrorAction Continue - } - exit 1 + # For devel builds/branches, use the dd-agent-mstesting bucket URL + # This allows us to build and test the package in PRs, and locally + # by using the `-VersionOverride` param. + if ([string]::IsNullOrEmpty($env:CI_PIPELINE_ID)) { + Write-Error "CI_PIPELINE_ID is not set, aborting" + exit 1 + } else { + if ($rawAgentVersion -notmatch $env:CI_PIPELINE_ID) { + Write-Error "CI_PIPELINE_ID is not found in the agent version, aborting" -ErrorAction Continue + if ([string]::IsNullOrEmpty($env:BUCKET_BRANCH)) { + # inv agent.version requires BUCKET_BRANCH to be set when including pipeline in version + Write-Error "BUCKET_BRANCH is not set, if you are running this locally, set `$env:BUCKET_BRANCH='dev' or pass the -VersionOverride parameter" -ErrorAction Continue } - $url = "https://s3.amazonaws.com/dd-agent-mstesting/pipelines/A7/$env:CI_PIPELINE_ID/$flavor-$rawAgentVersion-1-x86_64.msi" + exit 1 } + $url = "https://s3.amazonaws.com/dd-agent-mstesting/pipelines/A7/$env:CI_PIPELINE_ID/$flavor-$rawAgentVersion-1-x86_64.msi" } $agentVersionMatches = $rawAgentVersion | Select-String -Pattern $develPattern $agentVersion = "{0}-devel-{1}" -f $agentVersionMatches.Matches.Groups[1], $agentVersionMatches.Matches.Groups[2].Value @@ -151,26 +136,24 @@ try { exit 1 } - Write-Host "Generating Chocolatey $installMethod package $flavor version $agentVersion in $(Get-Location)" + Write-Host "Generating Chocolatey package $flavor version $agentVersion in $(Get-Location)" # Template the install script with the URL and checksum - if ($installMethod -eq "online") { - try { - $tempMsi = Join-Path -Path "$msiDirectory" "$flavor-$rawAgentVersion-1-x86_64.msi" - if (!(Test-Path $tempMsi)) { - Write-Host "Error: Could not find MSI file in $tempMsi" - Get-ChildItem "$msiDirectory" - exit 1 - } - $checksum = (Get-FileHash $tempMsi -Algorithm SHA256).Hash - } - catch { - Write-Host "Error: Could not generate checksum for package $($tempMsi): $($_)" + try { + $msiPath = Join-Path -Path "$msiDirectory" "$flavor-$rawAgentVersion-1-x86_64.msi" + if (!(Test-Path $msiPath)) { + Write-Host "Error: Could not find MSI file in $msiPath" + Get-ChildItem "$msiDirectory" exit 1 } - # Set the $url in the install script - (Get-Content $installScript).replace('$__url_from_ci__', '"' + $url + '"').replace('$__checksum_from_ci__', '"' + $checksum + '"') | Set-Content $installScript + $checksum = (Get-FileHash $msiPath -Algorithm SHA256).Hash + } + catch { + Write-Host "Error: Could not generate checksum for package $($msiPath): $($_)" + exit 1 } + # Set the $url in the install script + (Get-Content $installScript).replace('$__url_from_ci__', '"' + $url + '"').replace('$__checksum_from_ci__', '"' + $checksum + '"') | Set-Content $installScript Write-Host "Generated nuspec file:" Write-Host (Get-Content $installScript | Out-String) diff --git a/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 b/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 deleted file mode 100644 index e8879285da1ee..0000000000000 --- a/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -$ErrorActionPreference = 'Stop'; - -# Install chocolatey binary -$env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - -Set-Location c:\mnt -$nupkgs = Get-ChildItem .\nupkg\datadog-agent*.nupkg -foreach($nupkg in $nupkgs) { - Write-Host "Publishing Chocolatey package $($nupkg.Name) for agent version $agentVersion" - choco push $nupkg.FullName --verbose --key $env:CHOCOLATEY_API_KEY --source https://push.chocolatey.org/ - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } -} diff --git a/tasks/winbuildscripts/chocopush.bat b/tasks/winbuildscripts/chocopush.bat deleted file mode 100644 index d359e3c5c2dec..0000000000000 --- a/tasks/winbuildscripts/chocopush.bat +++ /dev/null @@ -1,11 +0,0 @@ -if not exist c:\mnt\ goto nomntdir - -@echo c:\mnt found, continuing - -Powershell -C "C:\mnt\tasks\winbuildscripts\Publish-Chocolatey-Package.ps1" || exit /b 1 -goto :EOF - -:nomntdir -@echo directory not mounted, parameters incorrect -exit /b 2 -goto :EOF \ No newline at end of file diff --git a/test/integration/serverless_perf/compute.sh b/test/integration/serverless_perf/compute.sh index c8421c24a84c9..06be249e90867 100755 --- a/test/integration/serverless_perf/compute.sh +++ b/test/integration/serverless_perf/compute.sh @@ -2,7 +2,7 @@ set -o pipefail -STARTUP_TIME_THRESHOLD=25 +STARTUP_TIME_THRESHOLD=28 calculate_median() { local sorted=($(printf "%s\n" "${@}" | sort -n)) diff --git a/test/new-e2e/examples/agentenv_checkruns_test.go b/test/new-e2e/examples/agentenv_checkruns_test.go index 7a7b73b77aab5..d62d0cfc05bc5 100644 --- a/test/new-e2e/examples/agentenv_checkruns_test.go +++ b/test/new-e2e/examples/agentenv_checkruns_test.go @@ -9,11 +9,12 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type agentSuiteEx5 struct { diff --git a/test/new-e2e/examples/agentenv_copyfolder_test.go b/test/new-e2e/examples/agentenv_copyfolder_test.go index 0aff0e3273db3..2c93589669d22 100644 --- a/test/new-e2e/examples/agentenv_copyfolder_test.go +++ b/test/new-e2e/examples/agentenv_copyfolder_test.go @@ -10,11 +10,12 @@ import ( "path" "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type agentSuiteEx6 struct { diff --git a/test/new-e2e/examples/agentenv_file_permissions_test.go b/test/new-e2e/examples/agentenv_file_permissions_test.go index a00fe133b4ba1..516498473859d 100644 --- a/test/new-e2e/examples/agentenv_file_permissions_test.go +++ b/test/new-e2e/examples/agentenv_file_permissions_test.go @@ -8,13 +8,14 @@ package examples import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" perms "github.com/DataDog/test-infra-definitions/components/datadog/agentparams/filepermissions" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) type filePermissionsTestSuite struct { diff --git a/test/new-e2e/examples/agentenv_file_permissions_win_test.go b/test/new-e2e/examples/agentenv_file_permissions_win_test.go index b35e4cceb003a..faf7fae0f2e96 100644 --- a/test/new-e2e/examples/agentenv_file_permissions_win_test.go +++ b/test/new-e2e/examples/agentenv_file_permissions_win_test.go @@ -17,7 +17,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/examples/agentenv_logs_test.go b/test/new-e2e/examples/agentenv_logs_test.go index 1f9ae6f347b34..0306d113c06de 100644 --- a/test/new-e2e/examples/agentenv_logs_test.go +++ b/test/new-e2e/examples/agentenv_logs_test.go @@ -13,7 +13,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" diff --git a/test/new-e2e/examples/agentenv_metrics_test.go b/test/new-e2e/examples/agentenv_metrics_test.go index 86150b5a5b26c..d3fb580e17f1e 100644 --- a/test/new-e2e/examples/agentenv_metrics_test.go +++ b/test/new-e2e/examples/agentenv_metrics_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/examples/agentenv_updateenv_test.go b/test/new-e2e/examples/agentenv_updateenv_test.go index b1f19447229b2..b13705226530e 100644 --- a/test/new-e2e/examples/agentenv_updateenv_test.go +++ b/test/new-e2e/examples/agentenv_updateenv_test.go @@ -8,10 +8,11 @@ package examples import ( "testing" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/examples/aks_test.go b/test/new-e2e/examples/aks_test.go index 4df8cfc7728c8..991999c300b81 100644 --- a/test/new-e2e/examples/aks_test.go +++ b/test/new-e2e/examples/aks_test.go @@ -17,7 +17,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - azurekubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/azure/kubernetes" + azurekubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/azure/kubernetes" ) type aksSuite struct { diff --git a/test/new-e2e/examples/azure_vm_test.go b/test/new-e2e/examples/azure_vm_test.go index 227b4b2ea28e7..64c59f9c156bf 100644 --- a/test/new-e2e/examples/azure_vm_test.go +++ b/test/new-e2e/examples/azure_vm_test.go @@ -6,9 +6,10 @@ package examples import ( - azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/azure/host/windows" "testing" + azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/azure/host/windows" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" ) diff --git a/test/new-e2e/examples/customenv_with_docker_app_test.go b/test/new-e2e/examples/customenv_with_docker_app_test.go index 8eb86a38054fe..71815e2a9631a 100644 --- a/test/new-e2e/examples/customenv_with_docker_app_test.go +++ b/test/new-e2e/examples/customenv_with_docker_app_test.go @@ -11,11 +11,13 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/test-infra-definitions/components/docker" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -47,7 +49,7 @@ var lighttpdConfigContent string //go:embed testfixtures/lighttpd_integration.conf.yaml var lighttpdIntegrationConfigContent string -func vmPlusDockerEnvProvisioner() e2e.PulumiEnvRunFunc[vmPlusDockerEnv] { +func vmPlusDockerEnvProvisioner() provisioners.PulumiEnvRunFunc[vmPlusDockerEnv] { return func(ctx *pulumi.Context, env *vmPlusDockerEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/examples/customenv_with_filemanager_test.go b/test/new-e2e/examples/customenv_with_filemanager_test.go index 219a36fdb0331..40938ba05c957 100644 --- a/test/new-e2e/examples/customenv_with_filemanager_test.go +++ b/test/new-e2e/examples/customenv_with_filemanager_test.go @@ -10,9 +10,11 @@ import ( "path" "testing" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" osComp "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -25,7 +27,7 @@ type fileManagerSuiteEx7 struct { e2e.BaseSuite[environments.Host] } -func customProvisionerFileManager(localFolderPath string, remoteFolderPath string) e2e.PulumiEnvRunFunc[environments.Host] { +func customProvisionerFileManager(localFolderPath string, remoteFolderPath string) provisioners.PulumiEnvRunFunc[environments.Host] { return func(ctx *pulumi.Context, env *environments.Host) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/examples/customenv_with_two_vm_test.go b/test/new-e2e/examples/customenv_with_two_vm_test.go index 4d45a35a62c04..c400927784c19 100644 --- a/test/new-e2e/examples/customenv_with_two_vm_test.go +++ b/test/new-e2e/examples/customenv_with_two_vm_test.go @@ -10,6 +10,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -23,7 +24,7 @@ type multiVMEnv struct { AppVM *components.RemoteHost } -func multiVMEnvProvisioner() e2e.PulumiEnvRunFunc[multiVMEnv] { +func multiVMEnvProvisioner() provisioners.PulumiEnvRunFunc[multiVMEnv] { return func(ctx *pulumi.Context, env *multiVMEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/examples/dockerenv_test.go b/test/new-e2e/examples/dockerenv_test.go index 22913d26f4251..729ea7b1d50a0 100644 --- a/test/new-e2e/examples/dockerenv_test.go +++ b/test/new-e2e/examples/dockerenv_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/examples/dockerenv_with_extra_compose_test.go b/test/new-e2e/examples/dockerenv_with_extra_compose_test.go index cf416c968b966..a99d8153bf3a3 100644 --- a/test/new-e2e/examples/dockerenv_with_extra_compose_test.go +++ b/test/new-e2e/examples/dockerenv_with_extra_compose_test.go @@ -10,13 +10,14 @@ import ( "strings" "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" "github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" ) type dockerSuiteWithExtraCompose struct { diff --git a/test/new-e2e/examples/ecs_test.go b/test/new-e2e/examples/ecs_test.go index 7a30eb6dfd7b4..e7e21ebb09c97 100644 --- a/test/new-e2e/examples/ecs_test.go +++ b/test/new-e2e/examples/ecs_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" tifEcs "github.com/DataDog/test-infra-definitions/scenarios/aws/ecs" awsecs "github.com/aws/aws-sdk-go-v2/service/ecs" diff --git a/test/new-e2e/examples/gcp_vm_test.go b/test/new-e2e/examples/gcp_vm_test.go index 1f897a87bbfe3..ec1d5fdb3a716 100644 --- a/test/new-e2e/examples/gcp_vm_test.go +++ b/test/new-e2e/examples/gcp_vm_test.go @@ -6,9 +6,10 @@ package examples import ( - gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/host/linux" "testing" + gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/host/linux" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" ) diff --git a/test/new-e2e/examples/gke_autopilot_test.go b/test/new-e2e/examples/gke_autopilot_test.go index 9317ace5b67be..82ced93302b4a 100644 --- a/test/new-e2e/examples/gke_autopilot_test.go +++ b/test/new-e2e/examples/gke_autopilot_test.go @@ -16,7 +16,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/kubernetes" + gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" diff --git a/test/new-e2e/examples/gke_test.go b/test/new-e2e/examples/gke_test.go index 5f01481914652..d2ffd1dcf444c 100644 --- a/test/new-e2e/examples/gke_test.go +++ b/test/new-e2e/examples/gke_test.go @@ -6,13 +6,15 @@ package examples import ( "context" - gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/kubernetes" + "strings" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strings" - "testing" + + gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" diff --git a/test/new-e2e/examples/kind_local_test.go b/test/new-e2e/examples/kind_local_test.go index dd5aa8f878c3a..a559f78be50a1 100644 --- a/test/new-e2e/examples/kind_local_test.go +++ b/test/new-e2e/examples/kind_local_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - localkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/local/kubernetes" + localkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/local/kubernetes" ) type myLocalKindSuite struct { diff --git a/test/new-e2e/examples/kind_test.go b/test/new-e2e/examples/kind_test.go index dcfa50dbc3f64..00677a5f673f9 100644 --- a/test/new-e2e/examples/kind_test.go +++ b/test/new-e2e/examples/kind_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/datadog/apps/dogstatsd" diff --git a/test/new-e2e/examples/vm_localpodman_test.go b/test/new-e2e/examples/vm_localpodman_test.go index 92941a6d7a8a4..c7157468e6219 100644 --- a/test/new-e2e/examples/vm_localpodman_test.go +++ b/test/new-e2e/examples/vm_localpodman_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - localhost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/local/host" + localhost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/local/host" ) type vmLocalPodmanSuite struct { diff --git a/test/new-e2e/examples/vm_test.go b/test/new-e2e/examples/vm_test.go index a2fea62f01969..d3d8375a4937d 100644 --- a/test/new-e2e/examples/vm_test.go +++ b/test/new-e2e/examples/vm_test.go @@ -10,7 +10,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type vmSuite struct { diff --git a/test/new-e2e/examples/vm_with_file_operations_test.go b/test/new-e2e/examples/vm_with_file_operations_test.go index 02f8592388bd6..d97764dac8daa 100644 --- a/test/new-e2e/examples/vm_with_file_operations_test.go +++ b/test/new-e2e/examples/vm_with_file_operations_test.go @@ -11,12 +11,13 @@ import ( "golang.org/x/crypto/ssh" + "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" - "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/examples/vmenv_withami_test.go b/test/new-e2e/examples/vmenv_withami_test.go index 1bf202cd16d86..eaae1d898408a 100644 --- a/test/new-e2e/examples/vmenv_withami_test.go +++ b/test/new-e2e/examples/vmenv_withami_test.go @@ -10,7 +10,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/os" diff --git a/test/new-e2e/examples/vmenv_withos_test.go b/test/new-e2e/examples/vmenv_withos_test.go index 6ec8775df5905..aa64c8483f8f0 100644 --- a/test/new-e2e/examples/vmenv_withos_test.go +++ b/test/new-e2e/examples/vmenv_withos_test.go @@ -8,12 +8,13 @@ package examples import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 008a6b32a4753..26734c2ec8418 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -45,7 +45,7 @@ replace ( require ( github.com/DataDog/agent-payload/v5 v5.0.138 github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/def v0.56.2 - github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/optional v0.59.1 github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 github.com/DataDog/datadog-agent/pkg/util/testutil v0.56.2 @@ -58,9 +58,9 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241211124138-9c7c5005ca28 - github.com/aws/aws-sdk-go-v2 v1.32.5 - github.com/aws/aws-sdk-go-v2/config v1.28.5 + github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 github.com/aws/aws-sdk-go-v2/service/eks v1.51.0 github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 @@ -72,10 +72,10 @@ require ( github.com/kr/pretty v0.3.1 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c github.com/pkg/sftp v1.13.7 - github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 - github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 - github.com/pulumi/pulumi/sdk/v3 v3.140.0 + github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 + github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 + github.com/pulumi/pulumi/sdk/v3 v3.142.0 github.com/samber/lo v1.47.0 github.com/stretchr/testify v1.10.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -109,22 +109,22 @@ require ( github.com/alessio/shellescape v1.4.2 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect - github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect @@ -171,7 +171,7 @@ require ( github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/hcl/v2 v2.22.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -217,8 +217,8 @@ require ( github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect github.com/pulumi/esc v0.10.0 // indirect github.com/pulumi/pulumi-command/sdk v1.0.1 // indirect - github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5 // indirect - github.com/pulumi/pulumi-libvirt/sdk v0.4.7 // indirect + github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7 // indirect + github.com/pulumi/pulumi-libvirt/sdk v0.5.3 // indirect github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 // indirect github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 // indirect github.com/pulumiverse/pulumi-time/sdk v0.1.0 // indirect @@ -253,7 +253,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/text v0.21.0 @@ -284,7 +284,7 @@ require ( github.com/DataDog/datadog-agent/pkg/trace v0.56.0-rc.3 github.com/DataDog/datadog-go/v5 v5.6.0 github.com/aws/aws-sdk-go v1.55.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/aws/session-manager-plugin v0.0.0-20241119210807-82dc72922492 github.com/digitalocean/go-libvirt v0.0.0-20240812180835-9c6c0a310c6c github.com/hairyhenderson/go-codeowners v0.7.0 @@ -309,7 +309,7 @@ require ( github.com/pulumi/pulumi-azure-native-sdk/managedidentity/v2 v2.73.1 // indirect github.com/pulumi/pulumi-azure-native-sdk/network/v2 v2.73.1 // indirect github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1 // indirect - github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0 // indirect + github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 // indirect github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 // indirect github.com/twinj/uuid v0.0.0-20151029044442-89173bcdda19 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index cbbd971dd8ac3..c79b919019bf8 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -17,8 +17,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241211124138-9c7c5005ca28 h1:LaZgAke+RN4wBKNl+R10ewdtKe/C2MJCbp9ozXKlLP8= -github.com/DataDog/test-infra-definitions v0.0.0-20241211124138-9c7c5005ca28/go.mod h1:blPG0VXBgk1oXm2+KHMTMyR0sNI2jv51FACAYPNQvNo= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 h1:w+uoG7RdtPiW6YvJvWrzWOC78eNzCemcH8ZM6tJoUBw= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266/go.mod h1:+13pRFKChJo9VZ0WcHFm5GTPCNYyim3hxFjy6cpcLG8= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= @@ -52,50 +52,50 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= -github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= -github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE= github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 h1:k97fGog9Tl0woxTiSIHN14Qs5ehqK6GXejUwkhJYyL0= github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0/go.mod h1:mzj8EEjIHSN2oZRXiw1Dd+uB4HZTl7hC8nBzX9IZMWw= -github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 h1:VDQaVwGOokbd3VUbHF+wupiffdrbAZPdQnr5XZMJqrs= -github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2/go.mod h1:lvUlMghKYmSxSfv0vU7pdU/8jSY+s0zpG8xXhaGKCw0= -github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4 h1:CTkPGE8fiElvLtYWl/U+Eu5+1fVXiZbJUjyVCRSRgxk= -github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4/go.mod h1:sMFLFhL27cKYa/eQYZp4asvIwHsnJWrAzTUpy9AQdnU= +github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7 h1:R+5XKIJga2K9Dkj0/iQ6fD/MBGo02oxGGFTc512lK/Q= +github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7/go.mod h1:fDPQV/6ONOQOjvtKhtypIy1wcGLcKYtoK/lvZ9fyDGQ= +github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2 h1:LRM6z+wmXqAgCvuH36RR+Wf8SZZhvOVjt6f5r38V2II= +github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2/go.mod h1:Ghi1OWUv4+VMEULWiHsKH2gNA3KAcMoLWsvU0eRXvIA= github.com/aws/aws-sdk-go-v2/service/eks v1.51.0 h1:BYyB+byjQ7oyupe3v+YjTp1yfmfNEwChYA2naCc85xI= github.com/aws/aws-sdk-go-v2/service/eks v1.51.0/go.mod h1:oaPCqTzAe8C5RQZJGRD4RENcV7A4n99uGxbD4rULbNg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 h1:z6Pq4+jtKlhK4wWJGHRGwMLGjC1HZwAO3KJr/Na0tSU= github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2/go.mod h1:DSmu/VZzpQlAubWBbAvNpt+S4k/XweglJi4XaDGyvQk= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= github.com/aws/session-manager-plugin v0.0.0-20241119210807-82dc72922492 h1:Ihams/fjKo4iWwM313ng2gCJWoetsL7ZQkXhOTmVUq4= github.com/aws/session-manager-plugin v0.0.0-20241119210807-82dc72922492/go.mod h1:7n17tunRPUsniNBu5Ja9C7WwJWTdOzaLqr/H0Ns3uuI= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= @@ -268,8 +268,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M= +github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -403,10 +403,10 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435 github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= github.com/pulumi/esc v0.10.0 h1:jzBKzkLVW0mePeanDRfqSQoCJ5yrkux0jIwAkUxpRKE= github.com/pulumi/esc v0.10.0/go.mod h1:2Bfa+FWj/xl8CKqRTWbWgDX0SOD4opdQgvYSURTGK2c= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 h1:wA38Ep4sEphX+3YGwFfaxRHs7NQv8dNObFepX6jaRa4= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1/go.mod h1:m/ejZ2INurqq/ncDjJfgC1Ff/lnbt0J/uO33BnPVots= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 h1:6082hB+ILpPB/0V5F+LTmHbX1BO54tCVOQCVOL/FYI4= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1/go.mod h1:z2bnBPHNYfk72IW1P01H9qikBtBSBhCwi3QpH6Y/38Q= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 h1:OvCLqUueOja9YE2WEGPYAw+lKHFRbLQ7QjwX55+uNsA= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0/go.mod h1:FFzye44v9E0BgaFXVB/9X7KH0S0MapoXEy2YonrQfz4= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 h1:jil2EBzZnKsRDrLfvx2gnAaq17HQLrTbpPsIb3h+98U= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0/go.mod h1:r+K4M7jnLqvvQDeR/0mBRq2EPZaqsDg24Ciy3ml/thA= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1 h1:miIJy4njnFYw7VxMLvEztoMPr9zYC2kqBTwRlaFAf48= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1/go.mod h1:LR1QBq0C1NIhmD9E0uKozCAu32j5qsamhrIsTSNVMS8= github.com/pulumi/pulumi-azure-native-sdk/compute/v2 v2.73.1 h1:79HTKSE1uJQolCRUHRFnIbSPNSIhxekIhznHnjpLi6s= @@ -421,22 +421,22 @@ github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1 h1:yzXxwwq3tHdtSOi5vjKmKXq7 github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1/go.mod h1:ChjIUNDNeN6jI33ZOivHUFqM6purDiLP01mghMGe1Fs= github.com/pulumi/pulumi-command/sdk v1.0.1 h1:ZuBSFT57nxg/fs8yBymUhKLkjJ6qmyN3gNvlY/idiN0= github.com/pulumi/pulumi-command/sdk v1.0.1/go.mod h1:C7sfdFbUIoXKoIASfXUbP/U9xnwPfxvz8dBpFodohlA= -github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5 h1:7OjAfgLz5PAy95ynbgPAlWls5WBe4I/QW/61TdPWRlQ= -github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5/go.mod h1:XZKLFXbw13olxuztlWnmVUPYZp2a+BqzqhuMl0j/Ow8= -github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0 h1:F3xAOBZ/In4PqydTsKeg3tou/c5FZ+JTp5dQO0oMjqE= -github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= +github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7 h1:cuIl5YyIghqtnFMGsdtPOeaNSix5S2CrqO0/UZ1Yjsc= +github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7/go.mod h1:f2ek887nKRSwNtqTqCFENJSOH0PXm1b3FhzSXYL0IyM= +github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 h1:s2Cpu6E2lmADNUbutbJGm6O+O9j0mBLlrhQmc40ukt0= +github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 h1:21oSj+TKlKTzQcxN9Hik7iSNNHPUQXN4s3itOnahy/w= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0/go.mod h1:YaEZms1NgXFqGhObKVofcAeWXu2V+3t/BAXdHQZq7fU= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 h1:quqoGsLbF7lpGpGU4mi5WfVLIAo4gfvoQeYYmemx1Dg= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3/go.mod h1:9dBA6+rtpKmyZB3k1XryUOHDOuNdoTODFKEEZZCtrz8= -github.com/pulumi/pulumi-libvirt/sdk v0.4.7 h1:/BBnqqx/Gbg2vINvJxXIVb58THXzw2lSqFqxlRSXH9M= -github.com/pulumi/pulumi-libvirt/sdk v0.4.7/go.mod h1:VKvjhAm1sGtzKZruYwIhgascabEx7+oVVRCoxp/cPi4= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 h1:7AjJpUyW6YHHpZr0bI6Fy1A3/b7ERxq1LAo5mlyNN1Y= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0/go.mod h1:ATS+UN8pguMxypQAK+SaPewesU+UN5dpf93PNqVuHzs= +github.com/pulumi/pulumi-libvirt/sdk v0.5.3 h1:CiUGTweLLIxbAbADxxnwPv4BK8pxXfU8urokJvK1ihM= +github.com/pulumi/pulumi-libvirt/sdk v0.5.3/go.mod h1:gAhyIZKtzs4rknrl8fu8BQnyqijAmViFbaUkyuHt4xY= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 h1:39rhOe/PTUGMYia8pR5T2wbxxMt2pwrlonf0ncYKSzE= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7/go.mod h1:cxxDhJzUPt/YElfvlWa15Q4NGF6XXS8kUs4OQsCxSBk= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 h1:tXemWrzeVTqG8zq6hBdv1TdPFXjgZ+dob63a/6GlF1o= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1/go.mod h1:hODo3iEmmXDFOXqPK+V+vwI0a3Ww7BLjs5Tgamp86Ng= -github.com/pulumi/pulumi/sdk/v3 v3.140.0 h1:+Z/RBvdYg7tBNkBwk4p/FzlV7niBT3TbLAICq/Y0LDU= -github.com/pulumi/pulumi/sdk/v3 v3.140.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= +github.com/pulumi/pulumi/sdk/v3 v3.142.0 h1:SmcVddGuvwAh3g3XUVQQ5gVRQUKH1yZ6iETpDNHIHlw= +github.com/pulumi/pulumi/sdk/v3 v3.142.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= github.com/pulumiverse/pulumi-time/sdk v0.1.0 h1:xfi9HKDgV+GgDxQ23oSv9KxC3DQqViGTcMrJICRgJv0= github.com/pulumiverse/pulumi-time/sdk v0.1.0/go.mod h1:NUa1zA74DF002WrM6iF111A6UjX9knPpXufVRvBwNyg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -595,8 +595,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= diff --git a/test/new-e2e/pkg/e2e/suite.go b/test/new-e2e/pkg/e2e/suite.go index bb15e92e7ce2e..586f87bb8cd76 100644 --- a/test/new-e2e/pkg/e2e/suite.go +++ b/test/new-e2e/pkg/e2e/suite.go @@ -154,6 +154,7 @@ import ( "github.com/DataDog/test-infra-definitions/components" "gopkg.in/zorkian/go-datadog-api.v2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra" @@ -175,7 +176,7 @@ type Suite[Env any] interface { init(params []SuiteOption, self Suite[Env]) - UpdateEnv(...Provisioner) + UpdateEnv(...provisioners.Provisioner) Env() *Env } @@ -189,8 +190,8 @@ type BaseSuite[Env any] struct { datadogClient *datadog.Client params suiteParams - originalProvisioners ProvisionerMap - currentProvisioners ProvisionerMap + originalProvisioners provisioners.ProvisionerMap + currentProvisioners provisioners.ProvisionerMap firstFailTest string startTime time.Time @@ -211,9 +212,9 @@ func (bs *BaseSuite[Env]) Env() *Env { } // UpdateEnv updates the environment with new provisioners. -func (bs *BaseSuite[Env]) UpdateEnv(newProvisioners ...Provisioner) { +func (bs *BaseSuite[Env]) UpdateEnv(newProvisioners ...provisioners.Provisioner) { uniqueIDs := make(map[string]struct{}) - targetProvisioners := make(ProvisionerMap, len(newProvisioners)) + targetProvisioners := make(provisioners.ProvisionerMap, len(newProvisioners)) for _, provisioner := range newProvisioners { if _, found := uniqueIDs[provisioner.ID()]; found { panic(fmt.Errorf("Multiple providers with same id found, provisioner with id %s already exists", provisioner.ID())) @@ -275,7 +276,7 @@ func (bs *BaseSuite[Env]) init(options []SuiteOption, self Suite[Env]) { bs.originalProvisioners = bs.params.provisioners } -func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error { +func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners provisioners.ProvisionerMap) error { if reflect.DeepEqual(bs.currentProvisioners, targetProvisioners) { bs.T().Logf("No change in provisioners, skipping environment update") return nil @@ -303,23 +304,23 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error } // Then we provision new resources - resources := make(RawResources) + resources := make(provisioners.RawResources) for id, provisioner := range targetProvisioners { - var provisionerResources RawResources + var provisionerResources provisioners.RawResources var err error bs.T().Logf("Provisioning environment stack %s with provisioner %s", bs.params.stackName, id) switch pType := provisioner.(type) { - case TypedProvisioner[Env]: + case provisioners.TypedProvisioner[Env]: provisionerResources, err = pType.ProvisionEnv(ctx, bs.params.stackName, logger, newEnv) - case UntypedProvisioner: + case provisioners.UntypedProvisioner: provisionerResources, err = pType.Provision(ctx, bs.params.stackName, logger) default: return fmt.Errorf("provisioner of type %T does not implement UntypedProvisioner nor TypedProvisioner", provisioner) } if err != nil { - if diagnosableProvisioner, ok := provisioner.(Diagnosable); ok { + if diagnosableProvisioner, ok := provisioner.(provisioners.Diagnosable); ok { stackName, err := infra.GetStackManager().GetPulumiStackName(bs.params.stackName) if err != nil { bs.T().Logf("unable to get stack name for diagnose, err: %v", err) @@ -359,7 +360,7 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error // On success we update the current environment // We need top copy provisioners to protect against external modifications - bs.currentProvisioners = copyProvisioners(targetProvisioners) + bs.currentProvisioners = provisioners.CopyProvisioners(targetProvisioners) bs.env = newEnv return nil } @@ -407,7 +408,7 @@ func (bs *BaseSuite[Env]) createEnv() (*Env, []reflect.StructField, []reflect.Va return &env, retainedFields, retainedValues, nil } -func (bs *BaseSuite[Env]) buildEnvFromResources(resources RawResources, fields []reflect.StructField, values []reflect.Value) error { +func (bs *BaseSuite[Env]) buildEnvFromResources(resources provisioners.RawResources, fields []reflect.StructField, values []reflect.Value) error { if len(fields) != len(values) { panic("fields and values must have the same length") } @@ -582,7 +583,7 @@ func (bs *BaseSuite[Env]) TearDownSuite() { for id, provisioner := range bs.originalProvisioners { // Run provisioner Diagnose before tearing down the stack - if diagnosableProvisioner, ok := provisioner.(Diagnosable); ok { + if diagnosableProvisioner, ok := provisioner.(provisioners.Diagnosable); ok { stackName, err := infra.GetStackManager().GetPulumiStackName(bs.params.stackName) if err != nil { bs.T().Logf("unable to get stack name for diagnose, err: %v", err) diff --git a/test/new-e2e/pkg/e2e/suite_params.go b/test/new-e2e/pkg/e2e/suite_params.go index bc38e26005a45..111da630670d2 100644 --- a/test/new-e2e/pkg/e2e/suite_params.go +++ b/test/new-e2e/pkg/e2e/suite_params.go @@ -8,8 +8,10 @@ package e2e import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" ) // Params implements [BaseSuite] options @@ -22,7 +24,7 @@ type suiteParams struct { skipDeleteOnFailure bool - provisioners ProvisionerMap + provisioners provisioners.ProvisionerMap } // SuiteOption is an optional function parameter type for e2e options @@ -53,14 +55,14 @@ func WithSkipDeleteOnFailure() SuiteOption { } // WithProvisioner adds a provisioner to the suite -func WithProvisioner(provisioner Provisioner) SuiteOption { +func WithProvisioner(provisioner provisioners.Provisioner) SuiteOption { return func(options *suiteParams) { if _, found := options.provisioners[provisioner.ID()]; found { panic(fmt.Sprintf("Duplicate provider in test Suite: %s", provisioner.ID())) } if options.provisioners == nil { - options.provisioners = make(ProvisionerMap) + options.provisioners = make(provisioners.ProvisionerMap) } options.provisioners[provisioner.ID()] = provisioner @@ -69,10 +71,10 @@ func WithProvisioner(provisioner Provisioner) SuiteOption { // WithUntypedPulumiProvisioner adds an untyped Pulumi provisioner to the suite func WithUntypedPulumiProvisioner(runFunc pulumi.RunFunc, configMap runner.ConfigMap) SuiteOption { - return WithProvisioner(NewUntypedPulumiProvisioner("", runFunc, configMap)) + return WithProvisioner(provisioners.NewUntypedPulumiProvisioner("", runFunc, configMap)) } // WithPulumiProvisioner adds a typed Pulumi provisioner to the suite -func WithPulumiProvisioner[Env any](runFunc PulumiEnvRunFunc[Env], configMap runner.ConfigMap) SuiteOption { - return WithProvisioner(NewTypedPulumiProvisioner("", runFunc, configMap)) +func WithPulumiProvisioner[Env any](runFunc provisioners.PulumiEnvRunFunc[Env], configMap runner.ConfigMap) SuiteOption { + return WithProvisioner(provisioners.NewTypedPulumiProvisioner("", runFunc, configMap)) } diff --git a/test/new-e2e/pkg/e2e/suite_test.go b/test/new-e2e/pkg/e2e/suite_test.go index 37219945b2389..e86152a30a395 100644 --- a/test/new-e2e/pkg/e2e/suite_test.go +++ b/test/new-e2e/pkg/e2e/suite_test.go @@ -14,6 +14,8 @@ import ( "github.com/DataDog/test-infra-definitions/components" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" ) type testTypeOutput struct { @@ -47,8 +49,8 @@ type testSuite struct { BaseSuite[testEnv] } -func testRawResources(key, value string) RawResources { - return RawResources{key: []byte(fmt.Sprintf(`{"myField":"%s"}`, value))} +func testRawResources(key, value string) provisioners.RawResources { + return provisioners.RawResources{key: []byte(fmt.Sprintf(`{"myField":"%s"}`, value))} } func TestCreateEnv(t *testing.T) { @@ -69,16 +71,16 @@ type testProvisioner struct { mock.Mock } -var _ UntypedProvisioner = &testProvisioner{} +var _ provisioners.UntypedProvisioner = &testProvisioner{} func (m *testProvisioner) ID() string { args := m.Called() return args.Get(0).(string) } -func (m *testProvisioner) Provision(arg0 context.Context, arg1 string, arg2 io.Writer) (RawResources, error) { +func (m *testProvisioner) Provision(arg0 context.Context, arg1 string, arg2 io.Writer) (provisioners.RawResources, error) { args := m.Called(arg0, arg1, arg2) - return args.Get(0).(RawResources), args.Error(1) + return args.Get(0).(provisioners.RawResources), args.Error(1) } func (m *testProvisioner) Destroy(arg0 context.Context, arg1 string, arg2 io.Writer) error { diff --git a/test/new-e2e/pkg/environments/doc.go b/test/new-e2e/pkg/environments/environments.go similarity index 100% rename from test/new-e2e/pkg/environments/doc.go rename to test/new-e2e/pkg/environments/environments.go diff --git a/test/new-e2e/pkg/environments/aws/docker/host.go b/test/new-e2e/pkg/provisioners/aws/docker/host.go similarity index 95% rename from test/new-e2e/pkg/environments/aws/docker/host.go rename to test/new-e2e/pkg/provisioners/aws/docker/host.go index 0e39ed8b74b20..0a730bdcf5e72 100644 --- a/test/new-e2e/pkg/environments/aws/docker/host.go +++ b/test/new-e2e/pkg/provisioners/aws/docker/host.go @@ -9,8 +9,8 @@ package awsdocker import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -224,10 +224,10 @@ func Run(ctx *pulumi.Context, env *environments.DockerHost, runParams RunParams) // Provisioner creates a VM environment with an EC2 VM with Docker, an ECS Fargate FakeIntake and a Docker Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.DockerHost] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.DockerHost] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.DockerHost) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.DockerHost) error { return Run(ctx, env, RunParams{ProvisionerParams: params}) }, params.extraConfigParams) diff --git a/test/new-e2e/pkg/environments/aws/ecs/ecs.go b/test/new-e2e/pkg/provisioners/aws/ecs/ecs.go similarity index 97% rename from test/new-e2e/pkg/environments/aws/ecs/ecs.go rename to test/new-e2e/pkg/provisioners/aws/ecs/ecs.go index 280920f2196a8..8e11e8a3ad0b9 100644 --- a/test/new-e2e/pkg/environments/aws/ecs/ecs.go +++ b/test/new-e2e/pkg/provisioners/aws/ecs/ecs.go @@ -28,8 +28,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ecs" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) @@ -300,10 +300,10 @@ func Run(ctx *pulumi.Context, env *environments.ECS, params *ProvisionerParams) // Provisioner creates a VM environment with an EC2 VM with Docker, an ECS Fargate FakeIntake and a Docker Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.ECS] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.ECS] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.ECS) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.ECS) error { return Run(ctx, env, params) }, params.extraConfigParams) diff --git a/test/new-e2e/pkg/environments/aws/host/host.go b/test/new-e2e/pkg/provisioners/aws/host/host.go similarity index 94% rename from test/new-e2e/pkg/environments/aws/host/host.go rename to test/new-e2e/pkg/provisioners/aws/host/host.go index 4668905ac83a8..ea652309c06e2 100644 --- a/test/new-e2e/pkg/environments/aws/host/host.go +++ b/test/new-e2e/pkg/provisioners/aws/host/host.go @@ -9,8 +9,8 @@ package awshost import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -151,7 +151,7 @@ func WithDocker() ProvisionerOption { } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -160,7 +160,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) @@ -283,11 +283,11 @@ func Run(ctx *pulumi.Context, env *environments.Host, runParams RunParams) error // Provisioner creates a VM environment with an EC2 VM, an ECS Fargate FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := GetProvisionerParams(opts...) diff --git a/test/new-e2e/pkg/environments/aws/host/windows/host.go b/test/new-e2e/pkg/provisioners/aws/host/windows/host.go similarity index 93% rename from test/new-e2e/pkg/environments/aws/host/windows/host.go rename to test/new-e2e/pkg/provisioners/aws/host/windows/host.go index 1fd5885a88c4d..642a1a7dcc2e2 100644 --- a/test/new-e2e/pkg/environments/aws/host/windows/host.go +++ b/test/new-e2e/pkg/provisioners/aws/host/windows/host.go @@ -19,8 +19,8 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -253,10 +253,10 @@ func getProvisionerParams(opts ...ProvisionerOption) *ProvisionerParams { // Provisioner creates a VM environment with a Windows EC2 VM, an ECS Fargate FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { // We need to build params here to be able to use params.name in the provisioner name params := getProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := getProvisionerParams(opts...) @@ -267,7 +267,7 @@ func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Wi } // ProvisionerNoAgent wraps Provisioner with hardcoded WithoutAgent options. -func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgent(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent()) @@ -276,7 +276,7 @@ func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environm } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -285,7 +285,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/eks.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/eks.go similarity index 95% rename from test/new-e2e/pkg/environments/aws/kubernetes/eks.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/eks.go index 22971ae20f9ed..1e44a7019a711 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/eks.go +++ b/test/new-e2e/pkg/provisioners/aws/kubernetes/eks.go @@ -26,8 +26,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/eks" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -42,13 +42,13 @@ func eksDiagnoseFunc(ctx context.Context, stackName string) (string, error) { } // EKSProvisioner creates a new provisioner -func EKSProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func EKSProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/kind.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/kind.go similarity index 95% rename from test/new-e2e/pkg/environments/aws/kubernetes/kind.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/kind.go index 60da0620167f9..44f5716bc92fa 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/kind.go +++ b/test/new-e2e/pkg/provisioners/aws/kubernetes/kind.go @@ -12,8 +12,8 @@ import ( "github.com/DataDog/test-infra-definitions/common/utils" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" "github.com/DataDog/test-infra-definitions/components/datadog/agent/helm" @@ -51,13 +51,13 @@ func kindDiagnoseFunc(ctx context.Context, stackName string) (string, error) { } // KindProvisioner creates a new provisioner -func KindProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func KindProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/kubernetes_dump.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/kubernetes_dump.go similarity index 100% rename from test/new-e2e/pkg/environments/aws/kubernetes/kubernetes_dump.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/kubernetes_dump.go diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/params.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/params.go similarity index 100% rename from test/new-e2e/pkg/environments/aws/kubernetes/params.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/params.go diff --git a/test/new-e2e/pkg/environments/azure/host/linux/host.go b/test/new-e2e/pkg/provisioners/azure/host/linux/host.go similarity index 92% rename from test/new-e2e/pkg/environments/azure/host/linux/host.go rename to test/new-e2e/pkg/provisioners/azure/host/linux/host.go index 5006c4c4750e3..30aa4cecedafa 100644 --- a/test/new-e2e/pkg/environments/azure/host/linux/host.go +++ b/test/new-e2e/pkg/provisioners/azure/host/linux/host.go @@ -7,13 +7,13 @@ package azurehost import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/azure" "github.com/DataDog/test-infra-definitions/scenarios/azure/compute" "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/components/datadog/agent" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" @@ -28,11 +28,11 @@ const ( // Provisioner creates a VM environment with an VM, a FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard-to-debug issues. params := GetProvisionerParams(opts...) diff --git a/test/new-e2e/pkg/environments/azure/host/linux/params.go b/test/new-e2e/pkg/provisioners/azure/host/linux/params.go similarity index 94% rename from test/new-e2e/pkg/environments/azure/host/linux/params.go rename to test/new-e2e/pkg/provisioners/azure/host/linux/params.go index ff32a326f06eb..923d4ac1778d8 100644 --- a/test/new-e2e/pkg/environments/azure/host/linux/params.go +++ b/test/new-e2e/pkg/provisioners/azure/host/linux/params.go @@ -7,15 +7,17 @@ package azurehost import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/resources/azure" "github.com/DataDog/test-infra-definitions/scenarios/azure/compute" "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) // ProvisionerParams is a set of parameters for the Provisioner. @@ -128,7 +130,7 @@ func WithUpdater() ProvisionerOption { } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -137,7 +139,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/azure/host/windows/host.go b/test/new-e2e/pkg/provisioners/azure/host/windows/host.go similarity index 89% rename from test/new-e2e/pkg/environments/azure/host/windows/host.go rename to test/new-e2e/pkg/provisioners/azure/host/windows/host.go index 414c6e8e469b0..dc32cff8ed9a4 100644 --- a/test/new-e2e/pkg/environments/azure/host/windows/host.go +++ b/test/new-e2e/pkg/provisioners/azure/host/windows/host.go @@ -7,7 +7,6 @@ package winazurehost import ( - installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" "github.com/DataDog/test-infra-definitions/components/activedirectory" "github.com/DataDog/test-infra-definitions/components/datadog/agent" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" @@ -16,7 +15,9 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/components/defender" ) @@ -28,10 +29,10 @@ const ( // Provisioner creates a VM environment with a Windows VM, a FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { // We need to build params here to be able to use params.name in the provisioner name params := getProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard-to-debug issues. params := getProvisionerParams(opts...) @@ -42,7 +43,7 @@ func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Wi } // ProvisionerNoAgent wraps Provisioner with hardcoded WithoutAgent options. -func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgent(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent()) @@ -51,7 +52,7 @@ func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environm } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -60,7 +61,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/azure/host/windows/params.go b/test/new-e2e/pkg/provisioners/azure/host/windows/params.go similarity index 100% rename from test/new-e2e/pkg/environments/azure/host/windows/params.go rename to test/new-e2e/pkg/provisioners/azure/host/windows/params.go diff --git a/test/new-e2e/pkg/environments/azure/kubernetes/aks.go b/test/new-e2e/pkg/provisioners/azure/kubernetes/aks.go similarity index 91% rename from test/new-e2e/pkg/environments/azure/kubernetes/aks.go rename to test/new-e2e/pkg/provisioners/azure/kubernetes/aks.go index 829a76675469c..53075f5f230fd 100644 --- a/test/new-e2e/pkg/environments/azure/kubernetes/aks.go +++ b/test/new-e2e/pkg/provisioners/azure/kubernetes/aks.go @@ -15,8 +15,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/azure/aks" "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) @@ -25,13 +25,13 @@ const ( ) // AKSProvisioner creates a new provisioner for AKS on Azure -func AKSProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func AKSProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/azure/kubernetes/params.go b/test/new-e2e/pkg/provisioners/azure/kubernetes/params.go similarity index 100% rename from test/new-e2e/pkg/environments/azure/kubernetes/params.go rename to test/new-e2e/pkg/provisioners/azure/kubernetes/params.go diff --git a/test/new-e2e/pkg/e2e/file_provisioner.go b/test/new-e2e/pkg/provisioners/file_provisioner.go similarity index 98% rename from test/new-e2e/pkg/e2e/file_provisioner.go rename to test/new-e2e/pkg/provisioners/file_provisioner.go index de8d0a4c22bd3..c0284816be6ef 100644 --- a/test/new-e2e/pkg/e2e/file_provisioner.go +++ b/test/new-e2e/pkg/provisioners/file_provisioner.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package e2e +package provisioners import ( "context" diff --git a/test/new-e2e/pkg/environments/gcp/host/linux/host.go b/test/new-e2e/pkg/provisioners/gcp/host/linux/host.go similarity index 92% rename from test/new-e2e/pkg/environments/gcp/host/linux/host.go rename to test/new-e2e/pkg/provisioners/gcp/host/linux/host.go index 0e479d8a51bdf..09e7be211d665 100644 --- a/test/new-e2e/pkg/environments/gcp/host/linux/host.go +++ b/test/new-e2e/pkg/provisioners/gcp/host/linux/host.go @@ -7,12 +7,12 @@ package gcphost import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/test-infra-definitions/resources/gcp" "github.com/DataDog/test-infra-definitions/scenarios/gcp/compute" "github.com/DataDog/test-infra-definitions/scenarios/gcp/fakeintake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/components/datadog/agent" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" @@ -27,11 +27,11 @@ const ( // Provisioner creates a VM environment with an VM, a FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard-to-debug issues. params := GetProvisionerParams(opts...) diff --git a/test/new-e2e/pkg/environments/gcp/host/linux/params.go b/test/new-e2e/pkg/provisioners/gcp/host/linux/params.go similarity index 94% rename from test/new-e2e/pkg/environments/gcp/host/linux/params.go rename to test/new-e2e/pkg/provisioners/gcp/host/linux/params.go index 442fd28b889b0..0542b3e045a66 100644 --- a/test/new-e2e/pkg/environments/gcp/host/linux/params.go +++ b/test/new-e2e/pkg/provisioners/gcp/host/linux/params.go @@ -7,15 +7,17 @@ package gcphost import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/resources/gcp" "github.com/DataDog/test-infra-definitions/scenarios/gcp/compute" "github.com/DataDog/test-infra-definitions/scenarios/gcp/fakeintake" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) // ProvisionerParams is a set of parameters for the Provisioner. @@ -128,7 +130,7 @@ func WithUpdater() ProvisionerOption { } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -137,7 +139,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/gcp/kubernetes/gke.go b/test/new-e2e/pkg/provisioners/gcp/kubernetes/gke.go similarity index 89% rename from test/new-e2e/pkg/environments/gcp/kubernetes/gke.go rename to test/new-e2e/pkg/provisioners/gcp/kubernetes/gke.go index 10c315fb7b254..80107a6369ce0 100644 --- a/test/new-e2e/pkg/environments/gcp/kubernetes/gke.go +++ b/test/new-e2e/pkg/provisioners/gcp/kubernetes/gke.go @@ -15,8 +15,8 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" "github.com/DataDog/test-infra-definitions/scenarios/gcp/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) @@ -25,13 +25,13 @@ const ( ) // GKEProvisioner creates a new provisioner for GKE on GCP -func GKEProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func GKEProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/gcp/kubernetes/params.go b/test/new-e2e/pkg/provisioners/gcp/kubernetes/params.go similarity index 100% rename from test/new-e2e/pkg/environments/gcp/kubernetes/params.go rename to test/new-e2e/pkg/provisioners/gcp/kubernetes/params.go diff --git a/test/new-e2e/pkg/environments/local/host/podman.go b/test/new-e2e/pkg/provisioners/local/host/podman.go similarity index 91% rename from test/new-e2e/pkg/environments/local/host/podman.go rename to test/new-e2e/pkg/provisioners/local/host/podman.go index 911a91f36ab76..85e73718dbcbe 100644 --- a/test/new-e2e/pkg/environments/local/host/podman.go +++ b/test/new-e2e/pkg/provisioners/local/host/podman.go @@ -7,8 +7,8 @@ package localhost import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -89,7 +89,7 @@ func WithExtraConfigParams(configMap runner.ConfigMap) ProvisionerOption { } // PodmanProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func PodmanProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func PodmanProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -98,7 +98,7 @@ func PodmanProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedPr } // PodmanProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func PodmanProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func PodmanProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) @@ -107,13 +107,13 @@ func PodmanProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvision } // PodmanProvisioner creates a new provisioner -func PodmanProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func PodmanProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/local/kubernetes/kind.go b/test/new-e2e/pkg/provisioners/local/kubernetes/kind.go similarity index 95% rename from test/new-e2e/pkg/environments/local/kubernetes/kind.go rename to test/new-e2e/pkg/provisioners/local/kubernetes/kind.go index 99703f64eb402..e754b6f57af91 100644 --- a/test/new-e2e/pkg/environments/local/kubernetes/kind.go +++ b/test/new-e2e/pkg/provisioners/local/kubernetes/kind.go @@ -9,8 +9,8 @@ package localkubernetes import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -105,13 +105,13 @@ func WithWorkloadApp(appFunc WorkloadAppFunc) ProvisionerOption { } // Provisioner creates a new provisioner -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/e2e/provisioner.go b/test/new-e2e/pkg/provisioners/provisioners.go similarity index 71% rename from test/new-e2e/pkg/e2e/provisioner.go rename to test/new-e2e/pkg/provisioners/provisioners.go index a537cc7a05ce4..7dd64c4468740 100644 --- a/test/new-e2e/pkg/e2e/provisioner.go +++ b/test/new-e2e/pkg/provisioners/provisioners.go @@ -3,7 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package e2e +// Package provisioners contains the definitions of the different provisioners that can be used in a test to setup an environment. +package provisioners import ( "context" @@ -21,6 +22,16 @@ type Provisioner interface { Destroy(context.Context, string, io.Writer) error } +// RawResources is the common types returned by provisioners +type RawResources map[string][]byte + +// Merge merges two RawResources maps +func (rr RawResources) Merge(in RawResources) { + for k, v := range in { + rr[k] = v + } +} + // UntypedProvisioner defines the interface for a provisioner without env binding type UntypedProvisioner interface { Provisioner @@ -36,7 +47,8 @@ type TypedProvisioner[Env any] interface { // ProvisionerMap is a map of provisioners. type ProvisionerMap map[string]Provisioner -func copyProvisioners(in ProvisionerMap) ProvisionerMap { +// CopyProvisioners copies a map of provisioners +func CopyProvisioners(in ProvisionerMap) ProvisionerMap { out := make(ProvisionerMap, len(in)) for k, v := range in { out[k] = v diff --git a/test/new-e2e/pkg/e2e/pulumi_provisioner.go b/test/new-e2e/pkg/provisioners/pulumi_provisioner.go similarity index 99% rename from test/new-e2e/pkg/e2e/pulumi_provisioner.go rename to test/new-e2e/pkg/provisioners/pulumi_provisioner.go index ef8aba2b8d295..ab9e6e69ca03c 100644 --- a/test/new-e2e/pkg/e2e/pulumi_provisioner.go +++ b/test/new-e2e/pkg/provisioners/pulumi_provisioner.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package e2e +package provisioners import ( "context" diff --git a/test/new-e2e/pkg/utils/infra/retriable_errors.go b/test/new-e2e/pkg/utils/infra/retriable_errors.go index b940cbcd2e064..211c9179511cb 100644 --- a/test/new-e2e/pkg/utils/infra/retriable_errors.go +++ b/test/new-e2e/pkg/utils/infra/retriable_errors.go @@ -55,5 +55,11 @@ func getKnownErrors() []knownError { errorMessage: `ssh: handshake failed: ssh: unable to authenticate`, retryType: ReCreate, }, + { + // https://datadoghq.atlassian.net/browse/ADXT-798 + // https://datadoghq.atlassian.net/browse/ADXT-813 + errorMessage: `error: awsx:ecs:FargateTaskDefinition resource '.+fakeintake.+' has a problem: grpc: the client connection is closing`, + retryType: ReCreate, + }, } } diff --git a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json b/test/new-e2e/system-probe/test-runner/files/cws_fentry.json deleted file mode 100644 index 8330f6d6048af..0000000000000 --- a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "filters": { - "*": { - "exclude": false - } - }, - "additional_env_vars": [ - "DD_EVENT_MONITORING_CONFIG_EVENT_STREAM_USE_FENTRY=true" - ] -} diff --git a/test/new-e2e/test-infra-definition/agent_test.go b/test/new-e2e/test-infra-definition/agent_test.go index 0452841024795..deae9e07d8916 100644 --- a/test/new-e2e/test-infra-definition/agent_test.go +++ b/test/new-e2e/test-infra-definition/agent_test.go @@ -8,11 +8,12 @@ package testinfradefinition import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type agentSuite struct { diff --git a/test/new-e2e/test-infra-definition/docker_test.go b/test/new-e2e/test-infra-definition/docker_test.go index 1ede9e48f9d66..a3c18fa65262d 100644 --- a/test/new-e2e/test-infra-definition/docker_test.go +++ b/test/new-e2e/test-infra-definition/docker_test.go @@ -12,12 +12,13 @@ import ( "testing" "time" + "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/stretchr/testify/assert" ) type dockerSuite struct { diff --git a/test/new-e2e/test-infra-definition/kind_test.go b/test/new-e2e/test-infra-definition/kind_test.go index 5312afb891f34..e31fa3c5c87f3 100644 --- a/test/new-e2e/test-infra-definition/kind_test.go +++ b/test/new-e2e/test-infra-definition/kind_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/datadog/apps/nginx" diff --git a/test/new-e2e/test-infra-definition/vm_test.go b/test/new-e2e/test-infra-definition/vm_test.go index a396b65a7eb25..0d5c098bc8d9b 100644 --- a/test/new-e2e/test-infra-definition/vm_test.go +++ b/test/new-e2e/test-infra-definition/vm_test.go @@ -10,7 +10,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/os" @@ -46,7 +47,7 @@ type vmSuiteWithUserData struct { type vmTestCase struct { testName string - provisioner e2e.Provisioner + provisioner provisioners.Provisioner suite e2e.Suite[environments.Host] } diff --git a/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go b/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go index c8f573f95307d..7e13406586659 100644 --- a/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/test-infra-definitions/components/datadog/apps/jmxfetch" diff --git a/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go b/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go index d66fa4f20746d..f863a3142877c 100644 --- a/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go +++ b/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go @@ -10,8 +10,8 @@ package kindfilelogging import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -112,13 +112,13 @@ func WithExtraConfigParams(configMap runner.ConfigMap) ProvisionerOption { } // Provisioner creates a new provisioner -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to issues that are hard to debug. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to issues that are hard to debug. params := newProvisionerParams() diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go index e2067a24f8cd7..9f442afa76893 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" ) diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go index 47c155ad6cc44..c8fb16ac43dde 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go @@ -17,7 +17,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" ) diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go index 1851a6e8d4ba6..d038e2624109b 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go @@ -17,7 +17,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) //go:embed config/utf-16-le.yaml diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go index bd70db87ca4c3..e29aaef83d61e 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go @@ -9,11 +9,12 @@ import ( _ "embed" "testing" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" ) type IntegrationsLogsSuite struct { diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go index 3a37f3ccda671..7472f18ad5d0d 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go @@ -15,11 +15,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" ) const eventuallyWithTickDuration = 5 * time.Second diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go index 6a11c2f7d2852..8dc817a250776 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" appslogger "github.com/DataDog/test-infra-definitions/components/datadog/apps/logger" "github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams" diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go index bd61e8cebde0d..f4bd94512b90e 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" ) diff --git a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go index 150dc1792462c..83ae2407654d4 100644 --- a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go +++ b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go b/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go index 0613fba20236b..52aa09e27a60e 100644 --- a/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go +++ b/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go @@ -10,13 +10,14 @@ import ( "fmt" "regexp" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/platforms" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - "github.com/stretchr/testify/assert" "testing" diff --git a/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go b/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go index 425f9b88d33c8..39fd4e5327ef1 100644 --- a/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go +++ b/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" diff --git a/test/new-e2e/tests/agent-platform/rpm/rpm_test.go b/test/new-e2e/tests/agent-platform/rpm/rpm_test.go index feeaaa5bfb5bf..19a5d221a58dd 100644 --- a/test/new-e2e/tests/agent-platform/rpm/rpm_test.go +++ b/test/new-e2e/tests/agent-platform/rpm/rpm_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go b/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go index 01ba9d04182a5..6c88086a0ab2b 100644 --- a/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go +++ b/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go b/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go index 06ecdf8d39273..1ac124114add3 100644 --- a/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go +++ b/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-shared-components/api/api_test.go b/test/new-e2e/tests/agent-shared-components/api/api_test.go index 97a0ee3553fa2..db7b22ffcdc49 100644 --- a/test/new-e2e/tests/agent-shared-components/api/api_test.go +++ b/test/new-e2e/tests/agent-shared-components/api/api_test.go @@ -26,7 +26,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go b/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go index 47c732d2569dc..356f76386e644 100644 --- a/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go +++ b/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go @@ -20,7 +20,7 @@ type agentConfigEndpointInfo struct { } func traceConfigEndpoint(port int) agentConfigEndpointInfo { - return agentConfigEndpointInfo{"trace-agent", "http", port, "/config"} + return agentConfigEndpointInfo{"trace-agent", "https", port, "/config"} } func processConfigEndpoint(port int) agentConfigEndpointInfo { diff --git a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go index 6ffb0a57472c6..d12913b0ea296 100644 --- a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go +++ b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" diff --git a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go index 5499a802a5e65..a68e94ec1d6fe 100644 --- a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go +++ b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" diff --git a/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go b/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go index 373bd61ce39af..12ef0c145b839 100644 --- a/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go +++ b/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go @@ -31,6 +31,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) @@ -82,7 +83,7 @@ var customLogsConfigTmplFile string //go:embed testfixtures/config.yaml.tmpl var configTmplFile string -func multiFakeIntakeAWS(agentOptions ...agentparams.Option) e2e.Provisioner { +func multiFakeIntakeAWS(agentOptions ...agentparams.Option) provisioners.Provisioner { runFunc := func(ctx *pulumi.Context, env *multiFakeIntakeEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -116,7 +117,7 @@ func multiFakeIntakeAWS(agentOptions ...agentparams.Option) e2e.Provisioner { return nil } - return e2e.NewTypedPulumiProvisioner("aws-nssfailover", runFunc, nil) + return provisioners.NewTypedPulumiProvisioner("aws-nssfailover", runFunc, nil) } type multiFakeIntakeSuite struct { diff --git a/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go b/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go index c0396429d896d..2f1656483cfe1 100644 --- a/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go +++ b/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" ) diff --git a/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go b/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go index b20b3fb5ce01e..c20ff3f1be9a3 100644 --- a/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go +++ b/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" ) diff --git a/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go b/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go index 5fa5c05dffc73..1e5ae1e964feb 100644 --- a/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go +++ b/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go b/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go index c06ca14096431..415493d5c7ebd 100644 --- a/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go +++ b/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go b/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go index b60e9a1bf594f..1cb11d89fffa0 100644 --- a/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go +++ b/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go b/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go index 0528334d553c0..f5bc2fbea85b0 100644 --- a/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go +++ b/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go b/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go index 253ecc8075cc3..69dde70ecfc02 100644 --- a/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-subcommands/check/check_win_test.go b/test/new-e2e/tests/agent-subcommands/check/check_win_test.go index 383663c37f0e6..c9398ad1e8d80 100644 --- a/test/new-e2e/tests/agent-subcommands/check/check_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/check/check_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsCheckSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/config/config_common_test.go b/test/new-e2e/tests/agent-subcommands/config/config_common_test.go index b7b289647a097..9c0c0e589f1d1 100644 --- a/test/new-e2e/tests/agent-subcommands/config/config_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/config/config_common_test.go @@ -10,14 +10,15 @@ import ( _ "embed" "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) type baseConfigSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go b/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go index 595f088a3d644..139bf58d4c2c2 100644 --- a/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go @@ -9,10 +9,11 @@ package config import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxConfigSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/config/config_win_test.go b/test/new-e2e/tests/agent-subcommands/config/config_win_test.go index d9b8684de8ec1..7d56eba5c8c82 100644 --- a/test/new-e2e/tests/agent-subcommands/config/config_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/config/config_win_test.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsConfigSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go index 6f6e744e09bcb..23df3f5210443 100644 --- a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxConfigCheckSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go index c82fa2c31f5fb..b53ea9629f151 100644 --- a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" diff --git a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go index bc32af4f6e4ba..88203b0db4931 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" svcmanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/svc-manager" ) diff --git a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go index bce881ffc8179..a8537e2887690 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsDiagnoseSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go b/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go index a027956ba8939..c0935a712cd28 100644 --- a/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go b/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go index 45c68f64a1507..144ffa170e54f 100644 --- a/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/stretchr/testify/assert" diff --git a/test/new-e2e/tests/agent-subcommands/health/health_common_test.go b/test/new-e2e/tests/agent-subcommands/health/health_common_test.go index 572282b7e1875..8aec70e72c28c 100644 --- a/test/new-e2e/tests/agent-subcommands/health/health_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/health/health_common_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/api" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/cenkalti/backoff" "github.com/stretchr/testify/assert" diff --git a/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go b/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go index 4a064a2d88f93..50e5ecef69919 100644 --- a/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxHealthSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/health/health_win_test.go b/test/new-e2e/tests/agent-subcommands/health/health_win_test.go index 84da19fdcadc5..33b0285dd31ea 100644 --- a/test/new-e2e/tests/agent-subcommands/health/health_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/health/health_win_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsHealthSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go index 7ddd73a1b1bdc..c22d8e08c4abe 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go @@ -17,7 +17,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/azure/host/linux" + azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/azure/host/linux" ) type linuxAzureHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go index f691a97c6bfd5..6636a8a00c0bb 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go @@ -6,11 +6,12 @@ package hostname import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type baseHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go index 2437c5a4306ad..f8e181b6608f2 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go @@ -8,13 +8,14 @@ package hostname import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) type linuxHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go index 3429569e29e7a..d07857f633922 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go index 72cdc02690437..540baa3fcfa4f 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/host/linux" + gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/host/linux" ) type linuxGCPHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go b/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go index 10376b4e30096..9bc5744a03765 100644 --- a/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/process" diff --git a/test/new-e2e/tests/agent-subcommands/run/run_win_test.go b/test/new-e2e/tests/agent-subcommands/run/run_win_test.go index 521eb1400fe07..04673cadd6934 100644 --- a/test/new-e2e/tests/agent-subcommands/run/run_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/run/run_win_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" ) diff --git a/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go b/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go index 69de472cab0f4..7050698056943 100644 --- a/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go @@ -10,7 +10,7 @@ import ( "time" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" diff --git a/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go b/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go index d4e099410dda6..bbeeef59ec1dd 100644 --- a/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go b/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go index ab805b2d90db7..35841716f6463 100644 --- a/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) diff --git a/test/new-e2e/tests/agent-subcommands/status/status_win_test.go b/test/new-e2e/tests/agent-subcommands/status/status_win_test.go index eef19d962c363..2a66687fe7303 100644 --- a/test/new-e2e/tests/agent-subcommands/status/status_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/status/status_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) diff --git a/test/new-e2e/tests/apm/docker_test.go b/test/new-e2e/tests/apm/docker_test.go index 04cdf2045e79f..4c7fc8451145f 100644 --- a/test/new-e2e/tests/apm/docker_test.go +++ b/test/new-e2e/tests/apm/docker_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" diff --git a/test/new-e2e/tests/apm/vm_test.go b/test/new-e2e/tests/apm/vm_test.go index 90f78554db9e2..006cd6353bd79 100644 --- a/test/new-e2e/tests/apm/vm_test.go +++ b/test/new-e2e/tests/apm/vm_test.go @@ -26,7 +26,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/containers/k8s_test.go b/test/new-e2e/tests/containers/k8s_test.go index 872714c799525..9561b71fb7a26 100644 --- a/test/new-e2e/tests/containers/k8s_test.go +++ b/test/new-e2e/tests/containers/k8s_test.go @@ -469,7 +469,10 @@ func (suite *k8sSuite) TestNginx() { `^pod_name:nginx-[[:alnum:]]+-[[:alnum:]]+$`, `^pod_phase:running$`, `^short_image:apps-nginx-server$`, - `^email:team-container-platform@datadoghq.com$`, + `^domain:deployment$`, + `^mail:team-container-platform@datadoghq.com$`, + `^org:agent-org$`, + `^parent-name:nginx$`, `^team:contp$`, }, AcceptUnexpectedTags: true, @@ -544,7 +547,10 @@ func (suite *k8sSuite) TestNginx() { `^pod_name:nginx-[[:alnum:]]+-[[:alnum:]]+$`, `^pod_phase:running$`, `^short_image:apps-nginx-server$`, - `^email:team-container-platform@datadoghq.com$`, + `^domain:deployment$`, + `^mail:team-container-platform@datadoghq.com$`, + `^org:agent-org$`, + `^parent-name:nginx$`, `^team:contp$`, }, Message: `GET / HTTP/1\.1`, @@ -795,6 +801,24 @@ func (suite *k8sSuite) TestCPU() { }) } +func (suite *k8sSuite) TestKSM() { + suite.testMetric(&testMetricArgs{ + Filter: testMetricFilterArgs{ + Name: "kubernetes_state.vpa.count", + }, + Expect: testMetricExpectArgs{ + Tags: &[]string{ + `^kube_cluster_name:` + regexp.QuoteMeta(suite.clusterName) + `$`, + `^kube_namespace:workload-(?:nginx|redis)$`, + }, + Value: &testMetricExpectValueArgs{ + Max: 1, + Min: 1, + }, + }, + }) +} + func (suite *k8sSuite) TestDogstatsdInAgent() { // Test with UDS suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDS) @@ -803,7 +827,7 @@ func (suite *k8sSuite) TestDogstatsdInAgent() { // Test with UDP + DD_ENTITY_ID suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDP) // Test with UDP + External Data - suite.testDogstatsdExternalData(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDPExternalData) + suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDPExternalData) } func (suite *k8sSuite) TestDogstatsdStandalone() { @@ -849,78 +873,6 @@ func (suite *k8sSuite) testDogstatsd(kubeNamespace, kubeDeployment string) { }) } -// testDogstatsdExternalData tests that the External Data origin resolution works for the dogstatsd. -func (suite *k8sSuite) testDogstatsdExternalData(kubeNamespace, kubeDeployment string) { - ctx := context.Background() - - // Record old pod, so we can be sure we are not looking at the incorrect one after deletion - oldPods, err := suite.K8sClient.CoreV1().Pods(kubeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - suite.Require().NoError(err) - suite.Require().Len(oldPods.Items, 1) - oldPod := oldPods.Items[0] - - // Delete the pod to ensure it is recreated after the admission controller is deployed - err = suite.K8sClient.CoreV1().Pods(kubeNamespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - suite.Require().NoError(err) - - // Wait for the fresh pod to be created - var pod corev1.Pod - suite.Require().EventuallyWithTf(func(c *assert.CollectT) { - pods, err := suite.K8sClient.CoreV1().Pods(kubeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - if !assert.NoError(c, err) { - return - } - if !assert.Len(c, pods.Items, 1) { - return - } - pod = pods.Items[0] - if !assert.NotEqual(c, oldPod.Name, pod.Name) { - return - } - }, 2*time.Minute, 10*time.Second, "Failed to witness the creation of pod with name %s in namespace %s", kubeDeployment, kubeNamespace) - - suite.Require().Len(pod.Spec.Containers, 1) - - suite.testMetric(&testMetricArgs{ - Filter: testMetricFilterArgs{ - Name: "custom.metric", - Tags: []string{ - "^kube_deployment:" + regexp.QuoteMeta(kubeDeployment) + "$", - "^kube_namespace:" + regexp.QuoteMeta(kubeNamespace) + "$", - }, - }, - Expect: testMetricExpectArgs{ - Tags: &[]string{ - `^container_id:`, - `^container_name:dogstatsd$`, - `^display_container_name:dogstatsd`, - `^git.commit.sha:`, // org.opencontainers.image.revision docker image label - `^git.repository_url:https://github.com/DataDog/test-infra-definitions$`, // org.opencontainers.image.source docker image label - `^image_id:ghcr.io/datadog/apps-dogstatsd@sha256:`, - `^image_name:ghcr.io/datadog/apps-dogstatsd$`, - `^image_tag:main$`, - `^kube_container_name:dogstatsd$`, - `^kube_deployment:` + regexp.QuoteMeta(kubeDeployment) + `$`, - "^kube_namespace:" + regexp.QuoteMeta(kubeNamespace) + "$", - `^kube_ownerref_kind:replicaset$`, - `^kube_ownerref_name:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+$`, - `^kube_qos:Burstable$`, - `^kube_replica_set:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+$`, - `^pod_name:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+-[[:alnum:]]+$`, - `^pod_phase:running$`, - `^series:`, - `^short_image:apps-dogstatsd$`, - }, - }, - }) -} - func (suite *k8sSuite) TestPrometheus() { // Test Prometheus check suite.testMetric(&testMetricArgs{ @@ -996,37 +948,12 @@ func (suite *k8sSuite) testAdmissionControllerPod(namespace string, name string, }, 5*time.Minute, 10*time.Second, "The deployment with name %s in namespace %s does not exist or does not have the auto detected languages annotation", name, namespace) } - // Record old pod, so we can be sure we are not looking at the incorrect one after deletion - oldPods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", name).String(), - }) - suite.Require().NoError(err) - suite.Require().Len(oldPods.Items, 1) - oldPod := oldPods.Items[0] - - // Delete the pod to ensure it is recreated after the admission controller is deployed - err = suite.K8sClient.CoreV1().Pods(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ + pods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ LabelSelector: fields.OneTermEqualSelector("app", name).String(), }) suite.Require().NoError(err) - - // Wait for the fresh pod to be created - var pod corev1.Pod - suite.Require().EventuallyWithTf(func(c *assert.CollectT) { - pods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", name).String(), - }) - if !assert.NoError(c, err) { - return - } - if !assert.Len(c, pods.Items, 1) { - return - } - pod = pods.Items[0] - if !assert.NotEqual(c, oldPod.Name, pod.Name) { - return - } - }, 2*time.Minute, 10*time.Second, "Failed to witness the creation of pod with name %s in namespace %s", name, namespace) + suite.Require().Len(pods.Items, 1) + pod := pods.Items[0] suite.Require().Len(pod.Spec.Containers, 1) diff --git a/test/new-e2e/tests/cspm/cspm_test.go b/test/new-e2e/tests/cspm/cspm_test.go index 6f5d881214691..929027e7595cd 100644 --- a/test/new-e2e/tests/cspm/cspm_test.go +++ b/test/new-e2e/tests/cspm/cspm_test.go @@ -24,7 +24,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" ) type cspmTestSuite struct { diff --git a/test/new-e2e/tests/cws/ec2_test.go b/test/new-e2e/tests/cws/ec2_test.go index 9266609f74ce0..c9fa1febb4efc 100644 --- a/test/new-e2e/tests/cws/ec2_test.go +++ b/test/new-e2e/tests/cws/ec2_test.go @@ -23,7 +23,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" "github.com/DataDog/datadog-agent/test/new-e2e/tests/cws/api" diff --git a/test/new-e2e/tests/cws/kind_test.go b/test/new-e2e/tests/cws/kind_test.go index 9dcfd5ef7da3b..31c93018509f7 100644 --- a/test/new-e2e/tests/cws/kind_test.go +++ b/test/new-e2e/tests/cws/kind_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/platforms" "github.com/DataDog/datadog-agent/test/new-e2e/tests/cws/api" diff --git a/test/new-e2e/tests/cws/windows_test.go b/test/new-e2e/tests/cws/windows_test.go index 4533befdf1653..ce840e6fe05dc 100644 --- a/test/new-e2e/tests/cws/windows_test.go +++ b/test/new-e2e/tests/cws/windows_test.go @@ -19,10 +19,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + testos "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - testos "github.com/DataDog/test-infra-definitions/components/os" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" diff --git a/test/new-e2e/tests/discovery/linux_test.go b/test/new-e2e/tests/discovery/linux_test.go index ec7d0aa339be3..a833bb7457346 100644 --- a/test/new-e2e/tests/discovery/linux_test.go +++ b/test/new-e2e/tests/discovery/linux_test.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) //go:embed testdata/config/agent_config.yaml @@ -170,7 +170,19 @@ func assertRunningCheck(t *assert.CollectT, remoteHost *components.RemoteHost, c func (s *linuxTestSuite) provisionServer() { err := s.Env().RemoteHost.CopyFolder("testdata/provision", "/home/ubuntu/e2e-test") require.NoError(s.T(), err) - s.Env().RemoteHost.MustExecute("sudo bash /home/ubuntu/e2e-test/provision.sh") + + cmd := "sudo bash /home/ubuntu/e2e-test/provision.sh" + _, err = s.Env().RemoteHost.Execute(cmd) + if err != nil { + // Sometimes temporary network errors are seen which cause the provision + // script to fail. + s.T().Log("Retrying provision due to failure", err) + time.Sleep(30 * time.Second) + _, err := s.Env().RemoteHost.Execute(cmd) + if err != nil { + s.T().Skip("Unable to provision server") + } + } } func (s *linuxTestSuite) startServices() { diff --git a/test/new-e2e/tests/gpu/provisioner.go b/test/new-e2e/tests/gpu/provisioner.go index ee552dcca7d8d..bc6db36edc6da 100644 --- a/test/new-e2e/tests/gpu/provisioner.go +++ b/test/new-e2e/tests/gpu/provisioner.go @@ -22,8 +22,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" ) // gpuEnabledAMI is an AMI that has GPU drivers pre-installed. In this case it's @@ -89,8 +89,8 @@ func getDefaultProvisionerParams() *provisionerParams { } } -func gpuInstanceProvisioner(params *provisionerParams) e2e.Provisioner { - return e2e.NewTypedPulumiProvisioner[environments.Host]("gpu", func(ctx *pulumi.Context, env *environments.Host) error { +func gpuInstanceProvisioner(params *provisionerParams) provisioners.Provisioner { + return provisioners.NewTypedPulumiProvisioner[environments.Host]("gpu", func(ctx *pulumi.Context, env *environments.Host) error { name := "gpuvm" awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/tests/ha-agent/haagent_test.go b/test/new-e2e/tests/ha-agent/haagent_test.go index cc173d2da7bae..12019455df4ab 100644 --- a/test/new-e2e/tests/ha-agent/haagent_test.go +++ b/test/new-e2e/tests/ha-agent/haagent_test.go @@ -20,7 +20,7 @@ import ( fakeintakeclient "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type haAgentTestSuite struct { diff --git a/test/new-e2e/tests/installer/script/all_scripts_test.go b/test/new-e2e/tests/installer/script/all_scripts_test.go index b79d99de89f1c..e50d01b0e18d7 100644 --- a/test/new-e2e/tests/installer/script/all_scripts_test.go +++ b/test/new-e2e/tests/installer/script/all_scripts_test.go @@ -12,14 +12,15 @@ import ( "strings" "testing" + e2eos "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" - e2eos "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - "github.com/stretchr/testify/require" ) type installerScriptTests func(os e2eos.Descriptor, arch e2eos.Architecture) installerScriptSuite diff --git a/test/new-e2e/tests/installer/script/databricks_test.go b/test/new-e2e/tests/installer/script/databricks_test.go index b6f038ab96803..820de6a627152 100644 --- a/test/new-e2e/tests/installer/script/databricks_test.go +++ b/test/new-e2e/tests/installer/script/databricks_test.go @@ -8,8 +8,9 @@ package installscript import ( "fmt" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" e2eos "github.com/DataDog/test-infra-definitions/components/os" + + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) const ( diff --git a/test/new-e2e/tests/installer/unix/all_packages_test.go b/test/new-e2e/tests/installer/unix/all_packages_test.go index f213a0fda8394..8fe357e74e741 100644 --- a/test/new-e2e/tests/installer/unix/all_packages_test.go +++ b/test/new-e2e/tests/installer/unix/all_packages_test.go @@ -21,7 +21,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" ) diff --git a/test/new-e2e/tests/installer/unix/package_agent_test.go b/test/new-e2e/tests/installer/unix/package_agent_test.go index 70ddeff75c676..d419ba8f02585 100644 --- a/test/new-e2e/tests/installer/unix/package_agent_test.go +++ b/test/new-e2e/tests/installer/unix/package_agent_test.go @@ -12,11 +12,12 @@ import ( "strings" "time" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" e2eos "github.com/DataDog/test-infra-definitions/components/os" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" + + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" ) const ( diff --git a/test/new-e2e/tests/installer/unix/package_installer_test.go b/test/new-e2e/tests/installer/unix/package_installer_test.go index 8588ee018d210..304cd12383e7b 100644 --- a/test/new-e2e/tests/installer/unix/package_installer_test.go +++ b/test/new-e2e/tests/installer/unix/package_installer_test.go @@ -6,9 +6,13 @@ package installer import ( - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "strings" + "time" + e2eos "github.com/DataDog/test-infra-definitions/components/os" "github.com/stretchr/testify/assert" + + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type packageInstallerSuite struct { @@ -102,20 +106,33 @@ func (s *packageInstallerSuite) TestReInstall() { func (s *packageInstallerSuite) TestUpdateInstallerOCI() { // Install prod err := s.RunInstallScriptProdOci( + "DD_REMOTE_UPDATES=true", envForceVersion("datadog-installer", "7.58.0-installer-0.5.1-1"), ) defer s.Purge() assert.NoError(s.T(), err) - version := s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") - assert.Equal(s.T(), "7.58.0-installer-0.5.1\n", version) + versionDisk := s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") + assert.Equal(s.T(), "7.58.0-installer-0.5.1\n", versionDisk) + assert.Eventually(s.T(), func() bool { + versionRunning, err := s.Env().RemoteHost.Execute("sudo datadog-installer status") + s.T().Logf("checking version: %s, err: %v", versionRunning, err) + return err == nil && strings.Contains(versionRunning, "7.58.0-installer-0.5.1") + }, 30*time.Second, 1*time.Second) // Install from QA registry - err = s.RunInstallScriptWithError() + err = s.RunInstallScriptWithError( + "DD_REMOTE_UPDATES=true", + ) assert.NoError(s.T(), err) - version = s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") - assert.NotEqual(s.T(), "7.58.0-installer-0.5.1\n", version) + versionDisk = s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") + assert.NotEqual(s.T(), "7.58.0-installer-0.5.1\n", versionDisk) + assert.Eventually(s.T(), func() bool { + versionRunning, err := s.Env().RemoteHost.Execute("sudo datadog-installer status") + s.T().Logf("checking version: %s, err: %v", versionRunning, err) + return err == nil && !strings.Contains(versionRunning, "7.58.0-installer-0.5.1") + }, 30*time.Second, 1*time.Second) } func (s *packageInstallerSuite) TestInstallWithUmask() { diff --git a/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go b/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go index 6230ef741410c..5fdf6a24c60eb 100644 --- a/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go +++ b/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go @@ -410,8 +410,8 @@ func (s *upgradeScenarioSuite) TestConfigUpgradeSuccessful() { state.AssertDirExists("/etc/datadog-agent/managed/datadog-agent", 0755, "root", "root") state.AssertSymlinkExists("/etc/datadog-agent/managed/datadog-agent/stable", "/etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d", "root", "root") // Verify metadata - state.AssertFileExists("/etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata", 0440, "dd-agent", "dd-agent") - file := s.Env().RemoteHost.MustExecute("sudo cat /etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata") + state.AssertFileExists("/etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata", 0444, "root", "root") + file := s.Env().RemoteHost.MustExecute("cat /etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata") policiesState := &pbgo.PoliciesState{} err := json.Unmarshal([]byte(file), policiesState) require.NoError(s.T(), err) diff --git a/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go b/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go index 9c7d4d716e468..6a1029294f235 100644 --- a/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go +++ b/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go @@ -8,13 +8,14 @@ package agenttests import ( installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - "github.com/stretchr/testify/require" "testing" + + "github.com/stretchr/testify/require" ) type testAgentInstallWithAgentUserSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go b/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go index 38fd858783940..a7c186c31cac2 100644 --- a/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go +++ b/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go @@ -10,7 +10,7 @@ import ( "path/filepath" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" diff --git a/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go b/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go index cad8fab208066..af28bc4d66b66 100644 --- a/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go +++ b/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installer "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/unix" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" ) diff --git a/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go b/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go index ad2ab623c5b57..b8aceac530e26 100644 --- a/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go +++ b/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go @@ -7,13 +7,14 @@ package agenttests import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - "github.com/stretchr/testify/require" "testing" + + "github.com/stretchr/testify/require" ) type testInstallScriptWithAgentUserSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go index 7bfa29e502797..9d2e4f4799351 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go @@ -7,11 +7,12 @@ package installertests import ( + "testing" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" - "testing" ) type testInstallerSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go index b211dce84ead1..8863214eca148 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go @@ -6,10 +6,11 @@ package installertests import ( + "testing" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" - "testing" ) type testInstallerRollbackSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go index 088bd9f80aee7..568e1159adc27 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go @@ -6,12 +6,13 @@ package installertests import ( + "testing" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" - "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/install-test" - "testing" + installtest "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/install-test" ) type testSystemIntegrityInstallerSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go index a2fc1d9970507..e145109d62b75 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go @@ -6,13 +6,14 @@ package installertests import ( + "testing" + agentVersion "github.com/DataDog/datadog-agent/pkg/version" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/pipeline" - "testing" ) type testInstallerUpgradesSuite struct { diff --git a/test/new-e2e/tests/language-detection/language_detection_test.go b/test/new-e2e/tests/language-detection/language_detection_test.go index 29ce07db52c7a..52d06d382ec5a 100644 --- a/test/new-e2e/tests/language-detection/language_detection_test.go +++ b/test/new-e2e/tests/language-detection/language_detection_test.go @@ -17,10 +17,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) //go:embed etc/process_config.yaml diff --git a/test/new-e2e/tests/language-detection/python_test.go b/test/new-e2e/tests/language-detection/python_test.go index 2be850b1da5a9..875ac519ce602 100644 --- a/test/new-e2e/tests/language-detection/python_test.go +++ b/test/new-e2e/tests/language-detection/python_test.go @@ -11,7 +11,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/require" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) func (s *languageDetectionSuite) installPython() { diff --git a/test/new-e2e/tests/ndm/netflow/netflow_test.go b/test/new-e2e/tests/ndm/netflow/netflow_test.go index 3694560cbbe44..cf2564c974db5 100644 --- a/test/new-e2e/tests/ndm/netflow/netflow_test.go +++ b/test/new-e2e/tests/ndm/netflow/netflow_test.go @@ -13,9 +13,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components/datadog/agent" @@ -37,8 +39,8 @@ var datadogYaml string // netflowDockerProvisioner defines a stack with a docker agent on an AmazonLinuxECS VM // with the netflow-generator running and sending netflow payloads to the agent -func netflowDockerProvisioner() e2e.Provisioner { - return e2e.NewTypedPulumiProvisioner[environments.DockerHost]("", func(ctx *pulumi.Context, env *environments.DockerHost) error { +func netflowDockerProvisioner() provisioners.Provisioner { + return provisioners.NewTypedPulumiProvisioner[environments.DockerHost]("", func(ctx *pulumi.Context, env *environments.DockerHost) error { name := "netflowvm" awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/tests/ndm/snmp/snmp_test.go b/test/new-e2e/tests/ndm/snmp/snmp_test.go index e4b2b56e53324..abdc91b3a69b8 100644 --- a/test/new-e2e/tests/ndm/snmp/snmp_test.go +++ b/test/new-e2e/tests/ndm/snmp/snmp_test.go @@ -13,11 +13,13 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components/datadog/agent" @@ -43,8 +45,8 @@ const ( // snmpDockerProvisioner defines a stack with a docker agent on an AmazonLinuxECS VM // with snmpsim installed and configured with snmp recordings -func snmpDockerProvisioner() e2e.Provisioner { - return e2e.NewTypedPulumiProvisioner("", func(ctx *pulumi.Context, env *environments.DockerHost) error { +func snmpDockerProvisioner() provisioners.Provisioner { + return provisioners.NewTypedPulumiProvisioner("", func(ctx *pulumi.Context, env *environments.DockerHost) error { name := "snmpvm" awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go index de5bc31f363ef..964382b9af13a 100644 --- a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go +++ b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go @@ -11,10 +11,11 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxNetworkPathIntegrationTestSuite struct { diff --git a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go index eb6efd844ab68..ae449f2b8ca5b 100644 --- a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go +++ b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go @@ -11,12 +11,13 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsNetworkPathIntegrationTestSuite struct { diff --git a/test/new-e2e/tests/npm/agentenv_npm_test.go b/test/new-e2e/tests/npm/agentenv_npm_test.go index 1dae47d257bfb..7fd062b40ac97 100644 --- a/test/new-e2e/tests/npm/agentenv_npm_test.go +++ b/test/new-e2e/tests/npm/agentenv_npm_test.go @@ -11,7 +11,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" diff --git a/test/new-e2e/tests/npm/ec2_1host_containerized_test.go b/test/new-e2e/tests/npm/ec2_1host_containerized_test.go index 68e1d68a92107..8fd4b25051591 100644 --- a/test/new-e2e/tests/npm/ec2_1host_containerized_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_containerized_test.go @@ -13,7 +13,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/test-infra-definitions/components/docker" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -30,7 +31,7 @@ type ec2VMContainerizedSuite struct { e2e.BaseSuite[dockerHostNginxEnv] } -func dockerHostHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[dockerHostNginxEnv] { +func dockerHostHttpbinEnvProvisioner() provisioners.PulumiEnvRunFunc[dockerHostNginxEnv] { return func(ctx *pulumi.Context, env *dockerHostNginxEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -76,7 +77,7 @@ func TestEC2VMContainerizedSuite(t *testing.T) { t.Parallel() s := &ec2VMContainerizedSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("dockerHostHttpbin", dockerHostHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("dockerHostHttpbin", dockerHostHttpbinEnvProvisioner(), nil))} // Source of our kitchen CI images test/kitchen/platforms.json // Other VM image can be used, our kitchen CI images test/kitchen/platforms.json diff --git a/test/new-e2e/tests/npm/ec2_1host_selinux_test.go b/test/new-e2e/tests/npm/ec2_1host_selinux_test.go index 67afaa540c9a4..2aaf28eb7138c 100644 --- a/test/new-e2e/tests/npm/ec2_1host_selinux_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_selinux_test.go @@ -9,7 +9,8 @@ import ( "testing" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" compos "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" @@ -25,7 +26,7 @@ func TestEC2VMSELinuxSuite(t *testing.T) { s := &ec2VMSELinuxSuite{} e2eParams := []e2e.SuiteOption{e2e.WithProvisioner( - e2e.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions( + provisioners.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions( // RHEL9 ec2.WithAMI("ami-0fe630eb857a6ec83", compos.AmazonLinux2, compos.AMD64Arch), ec2.WithInstanceType("t3.medium"))), nil)), } diff --git a/test/new-e2e/tests/npm/ec2_1host_test.go b/test/new-e2e/tests/npm/ec2_1host_test.go index ba07540f58ca7..78776323315a7 100644 --- a/test/new-e2e/tests/npm/ec2_1host_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_test.go @@ -13,7 +13,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/docker" @@ -30,7 +31,7 @@ type ec2VMSuite struct { e2e.BaseSuite[hostHttpbinEnv] } -func hostDockerHttpbinEnvProvisioner(opt ...awshost.ProvisionerOption) e2e.PulumiEnvRunFunc[hostHttpbinEnv] { +func hostDockerHttpbinEnvProvisioner(opt ...awshost.ProvisionerOption) provisioners.PulumiEnvRunFunc[hostHttpbinEnv] { return func(ctx *pulumi.Context, env *hostHttpbinEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -77,7 +78,7 @@ func TestEC2VMSuite(t *testing.T) { t.Parallel() s := &ec2VMSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(), nil))} // Source of our E2E CI images test/new-e2e/tests/agent-platform/platforms.json // Other VM image can be used, our E2E CI images test/new-e2e/tests/agent-platform/platforms.json diff --git a/test/new-e2e/tests/npm/ec2_1host_wkit_test.go b/test/new-e2e/tests/npm/ec2_1host_wkit_test.go index 1dd04c1541990..e6d02c3c07186 100644 --- a/test/new-e2e/tests/npm/ec2_1host_wkit_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_wkit_test.go @@ -12,7 +12,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type ec2VMWKitSuite struct { @@ -25,7 +26,7 @@ func TestEC2VMWKitSuite(t *testing.T) { s := &ec2VMWKitSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault))), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault))), nil))} e2e.Run(t, s, e2eParams...) } diff --git a/test/new-e2e/tests/npm/ecs_1host_test.go b/test/new-e2e/tests/npm/ecs_1host_test.go index ae617ac4040ae..c829ccdaa1b46 100644 --- a/test/new-e2e/tests/npm/ecs_1host_test.go +++ b/test/new-e2e/tests/npm/ecs_1host_test.go @@ -15,7 +15,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - envecs "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + envecs "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" npmtools "github.com/DataDog/test-infra-definitions/components/datadog/apps/npm-tools" "github.com/DataDog/test-infra-definitions/components/datadog/ecsagentparams" @@ -39,7 +40,7 @@ type ecsVMSuite struct { e2e.BaseSuite[ecsHttpbinEnv] } -func ecsHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[ecsHttpbinEnv] { +func ecsHttpbinEnvProvisioner() provisioners.PulumiEnvRunFunc[ecsHttpbinEnv] { return func(ctx *pulumi.Context, env *ecsHttpbinEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -86,7 +87,7 @@ func ecsHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[ecsHttpbinEnv] { func TestECSVMSuite(t *testing.T) { t.Parallel() s := &ecsVMSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("ecsHttpbin", ecsHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("ecsHttpbin", ecsHttpbinEnvProvisioner(), nil))} e2e.Run(t, s, e2eParams...) } diff --git a/test/new-e2e/tests/npm/eks_1host_test.go b/test/new-e2e/tests/npm/eks_1host_test.go index 12f86c5fd37e2..bfdd551266b86 100644 --- a/test/new-e2e/tests/npm/eks_1host_test.go +++ b/test/new-e2e/tests/npm/eks_1host_test.go @@ -22,7 +22,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - envkube "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + envkube "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" ) @@ -38,7 +39,7 @@ type eksVMSuite struct { e2e.BaseSuite[eksHttpbinEnv] } -func eksHttpbinEnvProvisioner(opts ...envkube.ProvisionerOption) e2e.PulumiEnvRunFunc[eksHttpbinEnv] { +func eksHttpbinEnvProvisioner(opts ...envkube.ProvisionerOption) provisioners.PulumiEnvRunFunc[eksHttpbinEnv] { return func(ctx *pulumi.Context, env *eksHttpbinEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -94,7 +95,7 @@ func TestEKSVMSuite(t *testing.T) { t.Parallel() s := &eksVMSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("eksHttpbin", eksHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("eksHttpbin", eksHttpbinEnvProvisioner(), nil))} e2e.Run(t, s, e2eParams...) } diff --git a/test/new-e2e/tests/orchestrator/apply.go b/test/new-e2e/tests/orchestrator/apply.go index ee4e8668ab40c..45b3d792102a7 100644 --- a/test/new-e2e/tests/orchestrator/apply.go +++ b/test/new-e2e/tests/orchestrator/apply.go @@ -18,6 +18,7 @@ import ( dogstatsdstandalone "github.com/DataDog/test-infra-definitions/components/datadog/dogstatsd-standalone" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" localKubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/DataDog/test-infra-definitions/components/kubernetes/vpa" resAws "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" @@ -86,6 +87,12 @@ func createCluster(ctx *pulumi.Context) (*resAws.Environment, *localKubernetes.C if err != nil { return nil, nil, nil, err } + + // Deploy VPA CRD + if _, err := vpa.DeployCRD(&awsEnv, kindKubeProvider); err != nil { + return nil, nil, nil, err + } + return &awsEnv, kindCluster, kindKubeProvider, nil } diff --git a/test/new-e2e/tests/otel/otel-agent/complete_test.go b/test/new-e2e/tests/otel/otel-agent/complete_test.go index d3359aae3cb87..37789d8b59766 100644 --- a/test/new-e2e/tests/otel/otel-agent/complete_test.go +++ b/test/new-e2e/tests/otel/otel-agent/complete_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/config/no-dd-exporter.yml b/test/new-e2e/tests/otel/otel-agent/config/no-dd-exporter.yml new file mode 100644 index 0000000000000..b952fe8d38811 --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/config/no-dd-exporter.yml @@ -0,0 +1,23 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go b/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go index 81b57d87917d9..2003c8c6e210e 100644 --- a/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go +++ b/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go b/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go index b0706f90e10ba..f97d9dfa62cb8 100644 --- a/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go +++ b/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/minimal_test.go b/test/new-e2e/tests/otel/otel-agent/minimal_test.go index 9377c56fd514f..ee33ec9d6cbac 100644 --- a/test/new-e2e/tests/otel/otel-agent/minimal_test.go +++ b/test/new-e2e/tests/otel/otel-agent/minimal_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) @@ -32,8 +32,8 @@ var minimalProvidedConfig string //go:embed testdata/minimal-full-config.yml var minimalFullConfig string -//go:embed testdata/minimal-sources.json -var minimalSources string +//go:embed testdata/sources.json +var sources string func TestOTelAgentMinimal(t *testing.T) { values := ` @@ -82,7 +82,7 @@ func (s *minimalTestSuite) TestOTelAgentInstalled() { } func (s *minimalTestSuite) TestOTelFlareExtensionResponse() { - utils.TestOTelFlareExtensionResponse(s, minimalProvidedConfig, minimalFullConfig, minimalSources) + utils.TestOTelFlareExtensionResponse(s, minimalProvidedConfig, minimalFullConfig, sources) } func (s *minimalTestSuite) TestOTelFlareFiles() { diff --git a/test/new-e2e/tests/otel/otel-agent/no_dd_exporter_test.go b/test/new-e2e/tests/otel/otel-agent/no_dd_exporter_test.go new file mode 100644 index 0000000000000..471e0c41d9dbd --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/no_dd_exporter_test.go @@ -0,0 +1,58 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package otelagent contains e2e otel agent tests +package otelagent + +import ( + _ "embed" + "testing" + + "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" + "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" +) + +type noDDExporterTestSuite struct { + e2e.BaseSuite[environments.Kubernetes] +} + +//go:embed config/no-dd-exporter.yml +var noDDExporterConfig string + +//go:embed testdata/no-dd-exporter-provided-config.yml +var noDDExporterProvidedConfig string + +//go:embed testdata/no-dd-exporter-full-config.yml +var noDDExporterFullConfig string + +func TestOTelAgentWithNoDDExporter(t *testing.T) { + values := ` +datadog: + logs: + containerCollectAll: false + containerCollectUsingFiles: false +` + t.Parallel() + e2e.Run(t, &noDDExporterTestSuite{}, + e2e.WithProvisioner( + awskubernetes.KindProvisioner( + awskubernetes.WithAgentOptions( + kubernetesagentparams.WithHelmValues(values), + kubernetesagentparams.WithOTelAgent(), + kubernetesagentparams.WithOTelConfig(noDDExporterConfig), + )))) +} + +func (s *noDDExporterTestSuite) TestOTelAgentInstalled() { + utils.TestOTelAgentInstalled(s) +} + +func (s *noDDExporterTestSuite) TestOTelFlareExtensionResponse() { + utils.TestOTelFlareExtensionResponse(s, noDDExporterProvidedConfig, noDDExporterFullConfig, sources) +} diff --git a/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go b/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go index aa9e8163e1664..6dee2b5052230 100644 --- a/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go +++ b/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/sampling_test.go b/test/new-e2e/tests/otel/otel-agent/sampling_test.go index 99f92bc32cc63..6fc08ce59c78b 100644 --- a/test/new-e2e/tests/otel/otel-agent/sampling_test.go +++ b/test/new-e2e/tests/otel/otel-agent/sampling_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-full-config.yml b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-full-config.yml new file mode 100644 index 0000000000000..e0276e1bb2aa3 --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-full-config.yml @@ -0,0 +1,281 @@ +connectors: {} +exporters: + datadog: + api: + fail_on_invalid_key: false + key: '[REDACTED]' + site: datadoghq.com + auth: null + compression: "" + cookies: null + disable_keep_alives: false + endpoint: "" + headers: {} + host_metadata: + enabled: false + hostname_source: config_or_system + reporter_period: 30m0s + tags: [] + hostname: "" + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: 1m30s + logs: + batch_wait: 5 + compression_level: 6 + dialer: + timeout: 0s + dump_payloads: false + use_compression: true + max_conns_per_host: 0 + max_idle_conns: 100 + max_idle_conns_per_host: 0 + metrics: + delta_ttl: 3600 + dialer: + timeout: 0s + histograms: + mode: distributions + send_aggregation_metrics: false + send_count_sum_metrics: false + instrumentation_scope_metadata_as_tags: false + resource_attributes_as_tags: false + summaries: + mode: gauges + sums: + cumulative_monotonic_mode: to_delta + initial_cumulative_monotonic_value: auto + only_metadata: false + proxy_url: "" + read_buffer_size: 0 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 1000 + storage: null + timeout: 15s + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + cipher_suites: [] + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + traces: + compute_stats_by_span_kind: false + compute_top_level_by_span_kind: true + dialer: + timeout: 0s + ignore_resources: [] + peer_service_aggregation: false + peer_tags: [] + peer_tags_aggregation: false + span_name_as_resource_name: false + span_name_remappings: {} + trace_buffer: 0 + write_buffer_size: 0 + debug: + sampling_initial: 2 + sampling_thereafter: 1 + use_internal_logger: true + verbosity: Detailed +extensions: + ddflare/dd-autoconfigured: + auth: null + compression_algorithms: [] + cors: null + endpoint: localhost:7777 + idle_timeout: 0s + include_metadata: false + max_request_body_size: 0 + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + write_timeout: 0s + health_check/dd-autoconfigured: + auth: null + check_collector_pipeline: + enabled: false + exporter_failure_threshold: 5 + interval: 5m + compression_algorithms: [] + cors: null + endpoint: localhost:13133 + idle_timeout: 0s + include_metadata: false + max_request_body_size: 0 + path: / + read_header_timeout: 0s + read_timeout: 0s + response_body: null + response_headers: {} + tls: null + write_timeout: 0s + pprof/dd-autoconfigured: + block_profile_fraction: 0 + dialer: + timeout: 0s + endpoint: localhost:1777 + mutex_profile_fraction: 0 + save_to_file: "" + zpages/dd-autoconfigured: + auth: null + compression_algorithms: [] + cors: null + endpoint: localhost:55679 + idle_timeout: 0s + include_metadata: false + max_request_body_size: 0 + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + write_timeout: 0s +processors: {} +receivers: + otlp: + protocols: + grpc: + auth: null + dialer: + timeout: 0s + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + compression_algorithms: [] + cors: null + endpoint: 0.0.0.0:4318 + idle_timeout: 0s + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + traces_url_path: /v1/traces + write_timeout: 0s + prometheus/dd-autoconfigured: + config: + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_protocols: + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + scrape_timeout: 10s + runtime: + gogc: 75 + scrape_configs: + - enable_compression: true + enable_http2: true + follow_redirects: true + honor_timestamps: true + job_name: datadog-agent + metrics_path: /metrics + scheme: http + scrape_interval: 10s + scrape_protocols: + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + scrape_timeout: 10s + static_configs: + - targets: + - 0.0.0.0:8888 + track_timestamps_staleness: false + report_extra_scrape_metrics: false + start_time_metric_regex: "" + target_allocator: null + trim_metric_suffixes: false + use_start_time_metric: false +service: + extensions: + - pprof/dd-autoconfigured + - zpages/dd-autoconfigured + - health_check/dd-autoconfigured + - ddflare/dd-autoconfigured + pipelines: + logs: + exporters: + - debug + processors: [] + receivers: + - otlp + metrics: + exporters: + - debug + processors: [] + receivers: + - otlp + metrics/dd-autoconfigured/datadog: + exporters: + - datadog + processors: [] + receivers: + - prometheus/dd-autoconfigured + traces: + exporters: + - debug + processors: [] + receivers: + - otlp + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: + - stderr + initial_fields: {} + level: info + output_paths: + - stderr + processors: [] + sampling: + enabled: true + initial: 10 + thereafter: 100 + tick: 10s + metrics: + address: "" + level: Normal + readers: + - pull: + exporter: + additionalproperties: null + prometheus: + host: localhost + port: 8888 + resource: {} + traces: + level: Basic + processors: [] + propagators: [] diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-provided-config.yml b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-provided-config.yml new file mode 100644 index 0000000000000..b97863a32c71c --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-provided-config.yml @@ -0,0 +1,184 @@ +connectors: {} +exporters: + datadog: + api: + fail_on_invalid_key: false + key: '[REDACTED]' + site: datadoghq.com + auth: null + compression: "" + cookies: null + disable_keep_alives: false + endpoint: "" + headers: {} + host_metadata: + enabled: false + hostname_source: config_or_system + reporter_period: 30m0s + tags: [] + hostname: "" + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: 1m30s + logs: + batch_wait: 5 + compression_level: 6 + dialer: + timeout: 0s + dump_payloads: false + use_compression: true + max_conns_per_host: 0 + max_idle_conns: 100 + max_idle_conns_per_host: 0 + metrics: + delta_ttl: 3600 + dialer: + timeout: 0s + histograms: + mode: distributions + send_aggregation_metrics: false + send_count_sum_metrics: false + instrumentation_scope_metadata_as_tags: false + resource_attributes_as_tags: false + summaries: + mode: gauges + sums: + cumulative_monotonic_mode: to_delta + initial_cumulative_monotonic_value: auto + only_metadata: false + proxy_url: "" + read_buffer_size: 0 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 1000 + storage: null + timeout: 15s + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + cipher_suites: [] + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + traces: + compute_stats_by_span_kind: false + compute_top_level_by_span_kind: true + dialer: + timeout: 0s + ignore_resources: [] + peer_service_aggregation: false + peer_tags: [] + peer_tags_aggregation: false + span_name_as_resource_name: false + span_name_remappings: {} + trace_buffer: 0 + write_buffer_size: 0 + debug: + sampling_initial: 2 + sampling_thereafter: 1 + use_internal_logger: true + verbosity: Detailed +extensions: {} +processors: {} +receivers: + otlp: + protocols: + grpc: + auth: null + dialer: + timeout: 0s + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + compression_algorithms: [] + cors: null + endpoint: 0.0.0.0:4318 + idle_timeout: 0s + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + traces_url_path: /v1/traces + write_timeout: 0s +service: + extensions: [] + pipelines: + logs: + exporters: + - debug + processors: [] + receivers: + - otlp + metrics: + exporters: + - debug + processors: [] + receivers: + - otlp + traces: + exporters: + - debug + processors: [] + receivers: + - otlp + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: + - stderr + initial_fields: {} + level: info + output_paths: + - stderr + processors: [] + sampling: + enabled: true + initial: 10 + thereafter: 100 + tick: 10s + metrics: + address: "" + level: Normal + readers: + - pull: + exporter: + additionalproperties: null + prometheus: + host: localhost + port: 8888 + resource: {} + traces: + level: Basic + processors: [] + propagators: [] \ No newline at end of file diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json b/test/new-e2e/tests/otel/otel-agent/testdata/sources.json similarity index 100% rename from test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json rename to test/new-e2e/tests/otel/otel-agent/testdata/sources.json diff --git a/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go b/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go index 89a30686706be..dff8498f51fa0 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go b/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go index 62b882e4def47..acbf90ad5a02a 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go b/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go index eb1e879491051..9be18d44c52e3 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go b/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go index 5f83e4dba06cf..74f5bcfa679d0 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/utils/config_utils.go b/test/new-e2e/tests/otel/utils/config_utils.go index ddee2793648fa..3d10d5267bc2e 100644 --- a/test/new-e2e/tests/otel/utils/config_utils.go +++ b/test/new-e2e/tests/otel/utils/config_utils.go @@ -211,13 +211,15 @@ func validateConfigs(t *testing.T, expectedCfg string, actualCfg string) { // Traces, metrics and logs endpoints are set dynamically to the fake intake address in the config // These endpoints vary from test to test and should be ignored in the comparison exps, _ := actualConfRaw["exporters"].(map[string]any) - ddExp, _ := exps["datadog"].(map[string]any) - tcfg := ddExp["traces"].(map[string]any) - delete(tcfg, "endpoint") - mcfg := ddExp["metrics"].(map[string]any) - delete(mcfg, "endpoint") - lcfg := ddExp["logs"].(map[string]any) - delete(lcfg, "endpoint") + if ddExp, ok := exps["datadog"]; ok { + ddExpCfg := ddExp.(map[string]any) + tcfg := ddExpCfg["traces"].(map[string]any) + delete(tcfg, "endpoint") + mcfg := ddExpCfg["metrics"].(map[string]any) + delete(mcfg, "endpoint") + lcfg := ddExpCfg["logs"].(map[string]any) + delete(lcfg, "endpoint") + } actualCfgBytes, err := yaml.Marshal(actualConfRaw) require.NoError(t, err) diff --git a/test/new-e2e/tests/process/docker_test.go b/test/new-e2e/tests/process/docker_test.go index c33a48f0a7473..536529b180750 100644 --- a/test/new-e2e/tests/process/docker_test.go +++ b/test/new-e2e/tests/process/docker_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" ) type dockerTestSuite struct { diff --git a/test/new-e2e/tests/process/ecs_test.go b/test/new-e2e/tests/process/ecs_test.go index 45881701fde06..a1c1e3d1b9e44 100644 --- a/test/new-e2e/tests/process/ecs_test.go +++ b/test/new-e2e/tests/process/ecs_test.go @@ -21,9 +21,10 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" ) type ECSEC2Suite struct { @@ -34,7 +35,7 @@ type ecsCPUStressEnv struct { environments.ECS } -func ecsEC2CPUStressProvisioner(runInCoreAgent bool) e2e.PulumiEnvRunFunc[ecsCPUStressEnv] { +func ecsEC2CPUStressProvisioner(runInCoreAgent bool) provisioners.PulumiEnvRunFunc[ecsCPUStressEnv] { return func(ctx *pulumi.Context, env *ecsCPUStressEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -65,7 +66,7 @@ func TestECSEC2TestSuite(t *testing.T) { t.Parallel() s := ECSEC2Suite{} e2eParams := []e2e.SuiteOption{e2e.WithProvisioner( - e2e.NewTypedPulumiProvisioner("ecsEC2CPUStress", ecsEC2CPUStressProvisioner(false), nil))} + provisioners.NewTypedPulumiProvisioner("ecsEC2CPUStress", ecsEC2CPUStressProvisioner(false), nil))} e2e.Run(t, &s, e2eParams...) } @@ -98,7 +99,7 @@ func TestECSEC2CoreAgentSuite(t *testing.T) { t.Parallel() s := ECSEC2CoreAgentSuite{} e2eParams := []e2e.SuiteOption{e2e.WithProvisioner( - e2e.NewTypedPulumiProvisioner("ecsEC2CoreAgentCPUStress", ecsEC2CPUStressProvisioner(true), nil))} + provisioners.NewTypedPulumiProvisioner("ecsEC2CoreAgentCPUStress", ecsEC2CPUStressProvisioner(true), nil))} e2e.Run(t, &s, e2eParams...) } diff --git a/test/new-e2e/tests/process/fargate_test.go b/test/new-e2e/tests/process/fargate_test.go index 695500ca7a8fe..11aa844f1170a 100644 --- a/test/new-e2e/tests/process/fargate_test.go +++ b/test/new-e2e/tests/process/fargate_test.go @@ -15,6 +15,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" @@ -25,14 +26,14 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" ) type ECSFargateSuite struct { e2e.BaseSuite[environments.ECS] } -func getFargateProvisioner(configMap runner.ConfigMap) e2e.TypedProvisioner[environments.ECS] { +func getFargateProvisioner(configMap runner.ConfigMap) provisioners.TypedProvisioner[environments.ECS] { return ecs.Provisioner( ecs.WithECSOptions(tifEcs.WithFargateCapacityProvider()), ecs.WithFargateWorkloadApp(func(e aws.Environment, clusterArn pulumi.StringInput, apiKeySSMParamName pulumi.StringInput, fakeIntake *fakeintakeComp.Fakeintake) (*ecsComp.Workload, error) { diff --git a/test/new-e2e/tests/process/k8s_test.go b/test/new-e2e/tests/process/k8s_test.go index 7ec7ccff16888..8d72e17fb535f 100644 --- a/test/new-e2e/tests/process/k8s_test.go +++ b/test/new-e2e/tests/process/k8s_test.go @@ -30,7 +30,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" ) // helmTemplate define the embedded minimal configuration for NPM diff --git a/test/new-e2e/tests/process/linux_test.go b/test/new-e2e/tests/process/linux_test.go index 7b5e093d5bf83..1934d33d6cbd6 100644 --- a/test/new-e2e/tests/process/linux_test.go +++ b/test/new-e2e/tests/process/linux_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxTestSuite struct { @@ -117,7 +117,7 @@ func (s *linuxTestSuite) TestProcessChecksInCoreAgent() { // Verify that the process agent is not running assert.EventuallyWithT(t, func(c *assert.CollectT) { - status := s.Env().RemoteHost.MustExecute("/opt/datadog-agent/embedded/bin/process-agent status") + status := s.Env().RemoteHost.MustExecute("sudo /opt/datadog-agent/embedded/bin/process-agent status") assert.Contains(c, status, "The Process Agent is not running") }, 1*time.Minute, 5*time.Second) diff --git a/test/new-e2e/tests/process/windows_test.go b/test/new-e2e/tests/process/windows_test.go index 5c7f7ef84711b..ba27cfc014703 100644 --- a/test/new-e2e/tests/process/windows_test.go +++ b/test/new-e2e/tests/process/windows_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsTestSuite struct { diff --git a/test/new-e2e/tests/remote-config/rc_ssl_config_test.go b/test/new-e2e/tests/remote-config/rc_ssl_config_test.go index 6fadad1dc47ff..b542f5fb18e36 100644 --- a/test/new-e2e/tests/remote-config/rc_ssl_config_test.go +++ b/test/new-e2e/tests/remote-config/rc_ssl_config_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type sslConfigSuite struct { diff --git a/test/new-e2e/tests/remote-config/tracer_test.go b/test/new-e2e/tests/remote-config/tracer_test.go index 6abc19e49c0ad..51150329e3caa 100644 --- a/test/new-e2e/tests/remote-config/tracer_test.go +++ b/test/new-e2e/tests/remote-config/tracer_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type tracerSuite struct { diff --git a/test/new-e2e/tests/security-agent-functional/security_agent_test.go b/test/new-e2e/tests/security-agent-functional/security_agent_test.go index 1b693e1b43b21..81f737f39f953 100644 --- a/test/new-e2e/tests/security-agent-functional/security_agent_test.go +++ b/test/new-e2e/tests/security-agent-functional/security_agent_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" ) diff --git a/test/new-e2e/tests/sysprobe-functional/apmtags_test.go b/test/new-e2e/tests/sysprobe-functional/apmtags_test.go index 3aff8ab90c5ae..7895154561a10 100644 --- a/test/new-e2e/tests/sysprobe-functional/apmtags_test.go +++ b/test/new-e2e/tests/sysprobe-functional/apmtags_test.go @@ -23,7 +23,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" ) diff --git a/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go b/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go index 888f9427e4413..710f3541de0fc 100644 --- a/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go +++ b/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go @@ -14,14 +14,15 @@ import ( "testing" "time" + componentsos "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - componentsos "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - "github.com/stretchr/testify/require" ) type vmSuite struct { diff --git a/test/new-e2e/tests/windows/domain-test/domain_test.go b/test/new-e2e/tests/windows/domain-test/domain_test.go index 134dc68d30683..47c779edf3acb 100644 --- a/test/new-e2e/tests/windows/domain-test/domain_test.go +++ b/test/new-e2e/tests/windows/domain-test/domain_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/activedirectory" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" "github.com/stretchr/testify/assert" diff --git a/test/new-e2e/tests/windows/fips-test/fips_test.go b/test/new-e2e/tests/windows/fips-test/fips_test.go index e46be55c5a941..48313099eed29 100644 --- a/test/new-e2e/tests/windows/fips-test/fips_test.go +++ b/test/new-e2e/tests/windows/fips-test/fips_test.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" diff --git a/test/new-e2e/tests/windows/install-test/base.go b/test/new-e2e/tests/windows/install-test/base.go index 566ee7c6225cc..958c171160f05 100644 --- a/test/new-e2e/tests/windows/install-test/base.go +++ b/test/new-e2e/tests/windows/install-test/base.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" diff --git a/test/new-e2e/tests/windows/install-test/installtester.go b/test/new-e2e/tests/windows/install-test/installtester.go index e0fbee6886600..f0c86fdf8c9d1 100644 --- a/test/new-e2e/tests/windows/install-test/installtester.go +++ b/test/new-e2e/tests/windows/install-test/installtester.go @@ -536,31 +536,6 @@ func (t *Tester) testInstalledFilePermissions(tt *testing.T, ddAgentUserIdentity }) } - // expect to have standard inherited permissions, plus an explciit ACE for ddagentuser - embeddedPaths := []string{ - filepath.Join(t.expectedInstallPath, "embedded3"), - } - if t.ExpectPython2Installed() { - embeddedPaths = append(embeddedPaths, - filepath.Join(t.expectedInstallPath, "embedded2"), - ) - } - agentUserFullAccessDirRule := windows.NewExplicitAccessRuleWithFlags( - ddAgentUserIdentity, - windows.FileFullControl, - windows.AccessControlTypeAllow, - windows.InheritanceFlagsContainer|windows.InheritanceFlagsObject, - windows.PropagationFlagsNone, - ) - for _, path := range embeddedPaths { - out, err := windows.GetSecurityInfoForPath(t.host, path) - require.NoError(tt, err) - if !windows.IsIdentityLocalSystem(ddAgentUserIdentity) { - windows.AssertContainsEqualable(tt, out.Access, agentUserFullAccessDirRule, "%s should have full access rule for %s", path, ddAgentUserIdentity) - } - assert.False(tt, out.AreAccessRulesProtected, "%s should inherit access rules", path) - } - // ensure the agent user does not have an ACE on the install dir out, err := windows.GetSecurityInfoForPath(t.host, t.expectedInstallPath) require.NoError(tt, err) diff --git a/test/new-e2e/tests/windows/powershell-module-test/install_test.go b/test/new-e2e/tests/windows/powershell-module-test/install_test.go index 1e289c567cb02..1bf62393b9e48 100644 --- a/test/new-e2e/tests/windows/powershell-module-test/install_test.go +++ b/test/new-e2e/tests/windows/powershell-module-test/install_test.go @@ -14,14 +14,15 @@ import ( "testing" "time" + "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" ) var ( diff --git a/test/new-e2e/tests/windows/service-test/startstop_test.go b/test/new-e2e/tests/windows/service-test/startstop_test.go index 837766c78f5e7..d4c3090f3d69d 100644 --- a/test/new-e2e/tests/windows/service-test/startstop_test.go +++ b/test/new-e2e/tests/windows/service-test/startstop_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" diff --git a/test/otel/go.mod b/test/otel/go.mod index fd0d05c30abe4..7689db77e10e6 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ./../../comp/core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ./../../comp/core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../comp/core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ./../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ./../../comp/def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../comp/forwarder/defaultforwarder @@ -129,6 +130,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect @@ -214,7 +216,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect @@ -292,7 +294,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index 270aa7b812248..a996d7f3007b6 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -168,8 +168,8 @@ github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= -github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -516,8 +516,8 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= diff --git a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml index b445834c940d6..0d3dba0eb57f2 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 30g + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml index c51505add2bb6..0d3dba0eb57f2 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 1.5GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml index c51505add2bb6..0d3dba0eb57f2 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 1.5GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml index ccca15933f243..a1b83cb9c0a98 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml index 215dcfa86cc48..b252bbe75d094 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 30g + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml index 68432ca735b1c..1db5b405767ff 100644 --- a/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml index 68432ca735b1c..1db5b405767ff 100644 --- a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_tree/experiment.yaml b/test/regression/cases/file_tree/experiment.yaml index 6c67c51ef14a5..1edff05390f25 100644 --- a/test/regression/cases/file_tree/experiment.yaml +++ b/test/regression/cases/file_tree/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/otel_to_otel_logs/experiment.yaml b/test/regression/cases/otel_to_otel_logs/experiment.yaml index dfbe54658435f..e4b260015988f 100644 --- a/test/regression/cases/otel_to_otel_logs/experiment.yaml +++ b/test/regression/cases/otel_to_otel_logs/experiment.yaml @@ -3,7 +3,6 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh cpu_allotment: 8 memory_allotment: 24GiB diff --git a/test/regression/cases/quality_gate_idle/experiment.yaml b/test/regression/cases/quality_gate_idle/experiment.yaml index 6555258f34601..d17ce5548c2fe 100644 --- a/test/regression/cases/quality_gate_idle/experiment.yaml +++ b/test/regression/cases/quality_gate_idle/experiment.yaml @@ -8,8 +8,7 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 + cpu_allotment: 4 memory_allotment: 500MiB environment: @@ -36,7 +35,7 @@ checks: description: "Memory usage quality gate. This puts a bound on the total agent memory usage." bounds: series: total_rss_bytes - upper_bound: "390.0 MiB" + upper_bound: "372.0 MiB" report_links: - text: "bounds checks dashboard" diff --git a/test/regression/cases/quality_gate_idle_all_features/experiment.yaml b/test/regression/cases/quality_gate_idle_all_features/experiment.yaml index 624b68caeab20..34e8e63fd4cf5 100644 --- a/test/regression/cases/quality_gate_idle_all_features/experiment.yaml +++ b/test/regression/cases/quality_gate_idle_all_features/experiment.yaml @@ -8,8 +8,7 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 + cpu_allotment: 4 memory_allotment: 1GiB environment: @@ -47,7 +46,7 @@ checks: description: "Memory usage quality gate. This puts a bound on the total agent memory usage." bounds: series: total_rss_bytes - upper_bound: "795.0 MiB" + upper_bound: "779.0 MiB" report_links: - text: "bounds checks dashboard" diff --git a/test/regression/cases/quality_gate_logs/experiment.yaml b/test/regression/cases/quality_gate_logs/experiment.yaml index b1b2d9ee9c02b..182f50e8679c4 100644 --- a/test/regression/cases/quality_gate_logs/experiment.yaml +++ b/test/regression/cases/quality_gate_logs/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 500MiB environment: DD_API_KEY: 00000001 @@ -29,7 +28,7 @@ checks: description: "Memory usage" bounds: series: total_rss_bytes - upper_bound: 440MiB + upper_bound: 429MiB - name: lost_bytes description: "Allowable bytes not polled by log Agent" diff --git a/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml b/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml index 17a5adb011dd1..461bbc607eb8b 100644 --- a/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml +++ b/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml b/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml index f9168734cf9e7..5de7351418822 100644 --- a/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml +++ b/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml b/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml index fe93432493ecf..b8e40dac08462 100644 --- a/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml +++ b/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml b/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml index ccb9f7b4e4866..c51909b2f72ee 100644 --- a/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml +++ b/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/config.yaml b/test/regression/config.yaml index 362301357f864..7a9684510426a 100644 --- a/test/regression/config.yaml +++ b/test/regression/config.yaml @@ -1,9 +1,7 @@ lading: - version: 0.24.0 + version: 0.25.2 target: - cpu_allotment: 8 - memory_allotment: 30g # Link templates for reports. # diff --git a/tools/windows/DatadogAgentInstaller/CustomActions/ConfigureUserCustomActions.cs b/tools/windows/DatadogAgentInstaller/CustomActions/ConfigureUserCustomActions.cs index 1447f8a080fd0..817f118b0c797 100644 --- a/tools/windows/DatadogAgentInstaller/CustomActions/ConfigureUserCustomActions.cs +++ b/tools/windows/DatadogAgentInstaller/CustomActions/ConfigureUserCustomActions.cs @@ -586,9 +586,6 @@ private List PathsWithAgentAccess() // agent needs to be able to write to run/ // agent needs to be able to create auth_token _session.Property("APPLICATIONDATADIRECTORY"), - // allow agent to write __pycache__ - Path.Combine(_session.Property("PROJECTLOCATION"), "embedded2"), - Path.Combine(_session.Property("PROJECTLOCATION"), "embedded3"), }; }