From 5615184aff643b2f613fdaad57b5aeabaea48d3a Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 18 Dec 2024 17:52:27 +0100 Subject: [PATCH 01/26] feat(fleet): Support integrations configuration in the Fleet Policies dir (#31263) --- cmd/agent/common/autodiscovery.go | 5 +++++ comp/core/flare/providers.go | 1 + comp/core/gui/guiimpl/checks.go | 17 ++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cmd/agent/common/autodiscovery.go b/cmd/agent/common/autodiscovery.go index 8ff855aeca19f..67a220b706750 100644 --- a/cmd/agent/common/autodiscovery.go +++ b/cmd/agent/common/autodiscovery.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "time" "go.uber.org/atomic" @@ -41,6 +42,10 @@ var ( ) func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, ac autodiscovery.Component) { + if pkgconfigsetup.Datadog().GetString("fleet_policies_dir") != "" { + confSearchPaths = append(confSearchPaths, filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d")) + } + providers.InitConfigFilesReader(confSearchPaths) acTelemetryStore := ac.GetTelemetryStore() diff --git a/comp/core/flare/providers.go b/comp/core/flare/providers.go index edb2c7ac86520..69ceae5ccf76f 100644 --- a/comp/core/flare/providers.go +++ b/comp/core/flare/providers.go @@ -53,6 +53,7 @@ func (f *flare) collectLogsFiles(fb types.FlareBuilder) error { func (f *flare) collectConfigFiles(fb types.FlareBuilder) error { confSearchPaths := map[string]string{ "": f.config.GetString("confd_path"), + "fleet": filepath.Join(f.config.GetString("fleet_policies_dir"), "conf.d"), "dist": filepath.Join(f.params.distPath, "conf.d"), "checksd": f.params.pythonChecksPath, } diff --git a/comp/core/gui/guiimpl/checks.go b/comp/core/gui/guiimpl/checks.go index 058a75b08f05c..f54c682f72b8b 100644 --- a/comp/core/gui/guiimpl/checks.go +++ b/comp/core/gui/guiimpl/checks.go @@ -42,9 +42,19 @@ var ( filepath.Join(defaultpaths.GetDistPath(), "checks.d"), // Custom checks pkgconfigsetup.Datadog().GetString("additional_checksd"), // Custom checks defaultpaths.PyChecksPath, // Integrations-core checks + getFleetPoliciesPath(), // Fleet Policies } ) +// getFleetPoliciesPath returns the path to the fleet policies directory if it is set in the configuration +// otherwise it returns an empty string +func getFleetPoliciesPath() string { + if len(pkgconfigsetup.Datadog().GetString("fleet_policies_dir")) > 0 { + return filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d") + } + return "" +} + // Adds the specific handlers for /checks/ endpoints func checkHandler(r *mux.Router, collector collector.Component, ac autodiscovery.Component) { r.HandleFunc("/running", http.HandlerFunc(sendRunningChecks)).Methods("POST") @@ -208,6 +218,9 @@ func getCheckConfigFile(w http.ResponseWriter, r *http.Request) { var file []byte var e error for _, path := range configPaths { + if len(path) == 0 { + continue + } filePath, err := securejoin.SecureJoin(path, fileName) if err != nil { log.Errorf("Error: Unable to join config path with the file name: %s", fileName) @@ -443,7 +456,9 @@ func getConfigsInPath(path string) ([]string, error) { func listConfigs(w http.ResponseWriter, _ *http.Request) { filenames := []string{} for _, path := range configPaths { - + if len(path) == 0 { + continue + } configs, e := getConfigsInPath(path) if e != nil { log.Errorf("Unable to list configurations from %s: %v", path, e) From 8f2d236ef1735bc576fb116495526de0221d1ff3 Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Wed, 18 Dec 2024 11:52:36 -0500 Subject: [PATCH 02/26] [Logs agent] Disable HTTP/2 when using a proxy (#32308) Co-authored-by: Jen Gilbert --- pkg/logs/client/http/destination.go | 7 +++++ pkg/logs/client/http/destination_test.go | 31 +++++++++++++++++++ ...-http1-for-log-agent-2cbfba763697ab42.yaml | 4 +-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/pkg/logs/client/http/destination.go b/pkg/logs/client/http/destination.go index 2a554e5476504..93bf35ec67167 100644 --- a/pkg/logs/client/http/destination.go +++ b/pkg/logs/client/http/destination.go @@ -396,6 +396,13 @@ func httpClientFactory(timeout time.Duration, cfg pkgconfigmodel.Reader) func() var transport *http.Transport transportConfig := cfg.Get("logs_config.http_protocol") + + // If any proxy is set, use http1 + // This will be removed in a future version + if cfg.GetProxies() != nil { + transportConfig = "http1" + } + // Configure transport based on user setting switch transportConfig { case "http1": diff --git a/pkg/logs/client/http/destination_test.go b/pkg/logs/client/http/destination_test.go index 92bf984a79e5a..3f65bba9b9e2c 100644 --- a/pkg/logs/client/http/destination_test.go +++ b/pkg/logs/client/http/destination_test.go @@ -494,3 +494,34 @@ func TestTransportProtocol_HTTP1FallBack(t *testing.T) { // Assert that the server automatically falls back to HTTP/1.1 assert.Equal(t, "HTTP/1.1", resp.Proto) } + +func TestTransportProtocol_HTTP1WhenUsingProxy(t *testing.T) { + c := configmock.New(t) + + // Force client to use ALNP + c.SetWithoutSource("logs_config.http_protocol", "auto") + c.SetWithoutSource("skip_ssl_validation", true) + + // The test server uses TLS, so if we set the http proxy (not https), it still makes + // a request to the test server, but disable HTTP/2 since a proxy is configured. + c.SetWithoutSource("proxy.http", "http://foo.bar") + + server := NewTestHTTPSServer(false) + defer server.Close() + + timeout := 5 * time.Second + client := httpClientFactory(timeout, c)() + + req, err := http.NewRequest("POST", server.URL, nil) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to send request: %v", err) + } + defer resp.Body.Close() + + // Assert that the server chose HTTP/1.1 because a proxy was configured + assert.Equal(t, "HTTP/1.1", resp.Proto) +} diff --git a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml index 53476dcfa0bb9..e9c1031ef5d3b 100644 --- a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml +++ b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml @@ -9,7 +9,7 @@ features: - | Introduced a new configuration variable `logs_config.http_protocol`, allowing users to enforce HTTP/1.1 for outgoing HTTP connections in the Datadog Agent. This provides better control over transport protocols and improves compatibility with systems that do not support HTTP/2. - By default, the log agent will now attempt to use HTTP/2 and fall back to the best available protocol if HTTP/2 is not supported. + By default, the log agent will now attempt to use HTTP/2 (unless a proxy is configured) and fall back to the best available protocol if HTTP/2 is not supported. enhancements: - | - Improved logging to add visiblity for latency and transport protocol \ No newline at end of file + Improved logging to add visibility for latency and transport protocol \ No newline at end of file From 33f08c8145777e0e0072188f6c4908ceb5dccb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lian=20Raimbault?= <161456554+CelianR@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:52:44 -0500 Subject: [PATCH 03/26] release.create-github-release: Use current version not next one (#32337) --- tasks/libs/releasing/version.py | 78 +++++++++++++++++++++++++++++++++ tasks/release.py | 67 ++-------------------------- 2 files changed, 81 insertions(+), 64 deletions(-) diff --git a/tasks/libs/releasing/version.py b/tasks/libs/releasing/version.py index 23472d57c0cca..85e66ea283080 100644 --- a/tasks/libs/releasing/version.py +++ b/tasks/libs/releasing/version.py @@ -6,6 +6,7 @@ from invoke import Exit from tasks.libs.ciproviders.github_api import GithubAPI +from tasks.libs.common.color import Color, color_message from tasks.libs.common.constants import ( AGENT_VERSION_CACHE_NAME, ALLOWED_REPO_NIGHTLY_BRANCHES, @@ -441,3 +442,80 @@ def get_matching_pattern(ctx, major_version, release=False): ) pattern = max(tags, key=cmp_to_key(semver.compare)) return pattern + + +def deduce_version(ctx, branch, as_str: bool = True, trust: bool = False, next_version: bool = True) -> str | Version: + """Deduces the version from the release branch name. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + """ + release_version = get_next_version_from_branch(ctx, branch, as_str=as_str, next_version=next_version) + + print( + f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr + ) + + if ( + trust + or not os.isatty(sys.stdin.fileno()) + or yes_no_question( + 'Is this the version you want to use?', + color="orange", + default=False, + ) + ): + return release_version + + raise Exit(color_message("Aborting.", "red"), code=1) + + +def get_version_major(branch: str) -> int: + """Get the major version from a branch name.""" + + return 7 if branch == 'main' else int(branch.split('.')[0]) + + +def get_all_version_tags(ctx) -> list[str]: + """Returns the tags for all the versions of the Agent in git.""" + + cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" + + return ctx.run(cmd, hide=True).stdout.strip().split('\n') + + +def get_next_version_from_branch(ctx, branch: str, as_str: bool = True, next_version: bool = True) -> str | Version: + """Returns the latest version + 1 belonging to a branch. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + + Example: + get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. + get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. + """ + + re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") + + try: + matched = re_branch.match(branch).group(1) + except Exception as e: + raise Exit( + f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 + ) from e + + tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] + versions = sorted(Version(*tag) for tag in tags) + + minor, major = tuple(map(int, branch.split('.')[:2])) + + if next_version: + # Get version after the latest one + version = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) + else: + # Get current latest version + assert versions, f"No tags found for branch {branch} (expected at least one tag)" + + version = versions[-1] + + return str(version) if as_str else version diff --git a/tasks/release.py b/tasks/release.py index 4e5805b0ffb76..c37e6e944deeb 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -70,10 +70,11 @@ VERSION_RE, _create_version_from_match, current_version, + deduce_version, + get_version_major, next_final_version, next_rc_version, ) -from tasks.libs.types.version import Version from tasks.pipeline import edit_schedule, run from tasks.release_metrics.metrics import get_prs_metrics, get_release_lead_time @@ -85,68 +86,6 @@ BACKPORT_LABEL_COLOR = "5319e7" -def deduce_version(ctx, branch, as_str=True, trust=False) -> str | Version: - release_version = get_next_version_from_branch(ctx, branch, as_str=as_str) - - print( - f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr - ) - - if ( - trust - or not os.isatty(sys.stdin.fileno()) - or yes_no_question( - 'Is this the version you want to use?', - color="orange", - default=False, - ) - ): - return release_version - - raise Exit(color_message("Aborting.", "red"), code=1) - - -def get_version_major(branch: str) -> int: - """Get the major version from a branch name.""" - - return 7 if branch == 'main' else int(branch.split('.')[0]) - - -def get_all_version_tags(ctx) -> list[str]: - """Returns the tags for all the versions of the Agent in git.""" - - cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" - - return ctx.run(cmd, hide=True).stdout.strip().split('\n') - - -def get_next_version_from_branch(ctx, branch: str, as_str=True) -> str | Version: - """Returns the latest version + 1 belonging to a branch. - - Example: - get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. - get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. - """ - - re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") - - try: - matched = re_branch.match(branch).group(1) - except Exception as e: - raise Exit( - f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 - ) from e - - tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] - versions = sorted(Version(*tag) for tag in tags) - - minor, major = tuple(map(int, branch.split('.')[:2])) - - latest = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) - - return str(latest) if as_str else latest - - @task def list_major_change(_, milestone): """List all PR labeled "major_changed" for this release.""" @@ -1221,7 +1160,7 @@ def create_github_release(ctx, release_branch, draft=True): ) notes = [] - version = deduce_version(ctx, release_branch) + version = deduce_version(ctx, release_branch, next_version=False) with agent_context(ctx, release_branch): for section, filename in sections: From de8114135d3a0c8a1bf806ef7959858dac31c95e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:54:58 +0000 Subject: [PATCH 04/26] Bump actions/checkout from 4.1.4 to 4.2.2 (#30957) Co-authored-by: chouetz --- .github/workflows/add_milestone.yml | 2 +- .github/workflows/buildimages-update.yml | 4 ++-- .github/workflows/chase_release_managers.yml | 2 +- .github/workflows/code_review_complexity.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/create_rc_pr.yml | 4 ++-- .github/workflows/create_release_schedule.yml | 2 +- .github/workflows/cws-btfhub-sync.yml | 6 +++--- .github/workflows/datadog-static-analysis.yml | 2 +- .github/workflows/docs-dev.yml | 2 +- .github/workflows/external-contributor.yml | 2 +- .github/workflows/go-update-commenter.yml | 4 ++-- .github/workflows/go_mod_tidy.yml | 2 +- .github/workflows/gohai.yml | 2 +- .github/workflows/label-analysis.yml | 6 +++--- .github/workflows/markdown-lint-check.yml | 2 +- .github/workflows/report-merged-pr.yml | 2 +- .github/workflows/serverless-benchmarks.yml | 4 ++-- .github/workflows/serverless-binary-size.yml | 4 ++-- .github/workflows/serverless-integration.yml | 4 ++-- 20 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/add_milestone.yml b/.github/workflows/add_milestone.yml index 33885369f0e0c..6bcd28e967c2e 100644 --- a/.github/workflows/add_milestone.yml +++ b/.github/workflows/add_milestone.yml @@ -22,7 +22,7 @@ jobs: GH_REPO: ${{ github.repository }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml index 1138e8557d011..a480105bf17e8 100644 --- a/.github/workflows/buildimages-update.yml +++ b/.github/workflows/buildimages-update.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # credentials are needed to create the PR at the end of the workflow persist-credentials: true @@ -53,7 +53,7 @@ jobs: fi - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: ${{ steps.branch_fetch.outputs.RESULT == 'true' }} with: ref: ${{ inputs.branch }} diff --git a/.github/workflows/chase_release_managers.yml b/.github/workflows/chase_release_managers.yml index 436217659ae48..475068736efea 100644 --- a/.github/workflows/chase_release_managers.yml +++ b/.github/workflows/chase_release_managers.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/code_review_complexity.yml b/.github/workflows/code_review_complexity.yml index 8c26254f6c626..3f36b387e0f34 100644 --- a/.github/workflows/code_review_complexity.yml +++ b/.github/workflows/code_review_complexity.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bb6d39efa5e6d..bc52cb502bdaf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 2a685c3cc85df..66fbf9462a168 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -21,7 +21,7 @@ jobs: warning: ${{ steps.warning.outputs.value }} steps: - name: Checkout repository - if: ${{ env.IS_AGENT6_RELEASE == 'false' }} + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: sparse-checkout: 'tasks' @@ -69,7 +69,7 @@ jobs: fail-fast: false steps: - name: Checkout the main branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: true diff --git a/.github/workflows/create_release_schedule.yml b/.github/workflows/create_release_schedule.yml index f8dc006160e3c..4b749ba3bdc97 100644 --- a/.github/workflows/create_release_schedule.yml +++ b/.github/workflows/create_release_schedule.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index 795953b22a354..94498f5ed7525 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -52,13 +52,13 @@ jobs: df -h - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} persist-credentials: false - name: Checkout btfhub-archive repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/btfhub-archive path: dev/dist/archive @@ -110,7 +110,7 @@ jobs: pull-requests: write steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml index 3063c7c0db456..a77f4ba632c1d 100644 --- a/.github/workflows/datadog-static-analysis.yml +++ b/.github/workflows/datadog-static-analysis.yml @@ -11,7 +11,7 @@ jobs: name: Datadog Static Analyzer steps: - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Check code meets quality and security standards diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 1eab6460db4b6..59e803598f6cd 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false # Fetch all history for applying timestamps to every page diff --git a/.github/workflows/external-contributor.yml b/.github/workflows/external-contributor.yml index f0beb805c2a50..03cf46fd03511 100644 --- a/.github/workflows/external-contributor.yml +++ b/.github/workflows/external-contributor.yml @@ -17,7 +17,7 @@ jobs: if: github.event.pull_request.head.repo.full_name != github.repository steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: main fetch-depth: 0 diff --git a/.github/workflows/go-update-commenter.yml b/.github/workflows/go-update-commenter.yml index 1028110bc4fda..c74a404a09c4c 100644 --- a/.github/workflows/go-update-commenter.yml +++ b/.github/workflows/go-update-commenter.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: # get the Go version of the target branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -26,7 +26,7 @@ jobs: echo version="$(cat .go-version)" >> $GITHUB_OUTPUT # get the Go version of the PR branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Get current Go version diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml index e48d806c9dce9..4cc1a55690e6f 100644 --- a/.github/workflows/go_mod_tidy.yml +++ b/.github/workflows/go_mod_tidy.yml @@ -17,7 +17,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Checkout PR diff --git a/.github/workflows/gohai.yml b/.github/workflows/gohai.yml index 851c2f9d0ccd1..7ab3b5e49d2fc 100644 --- a/.github/workflows/gohai.yml +++ b/.github/workflows/gohai.yml @@ -34,7 +34,7 @@ jobs: go-file: [.go-version, pkg/gohai/go.mod] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 diff --git a/.github/workflows/label-analysis.yml b/.github/workflows/label-analysis.yml index ec53d5a695fed..7c4345e2ec323 100644 --- a/.github/workflows/label-analysis.yml +++ b/.github/workflows/label-analysis.yml @@ -23,7 +23,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false @@ -125,7 +125,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Setup python diff --git a/.github/workflows/markdown-lint-check.yml b/.github/workflows/markdown-lint-check.yml index 5f440614390ae..b5fe87f21f0f8 100644 --- a/.github/workflows/markdown-lint-check.yml +++ b/.github/workflows/markdown-lint-check.yml @@ -9,7 +9,7 @@ jobs: markdown-link-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: gaurav-nelson/github-action-markdown-link-check@d53a906aa6b22b8979d33bc86170567e619495ec # v1.0.15 diff --git a/.github/workflows/report-merged-pr.yml b/.github/workflows/report-merged-pr.yml index feefb3c5446bd..879b5b7e13330 100644 --- a/.github/workflows/report-merged-pr.yml +++ b/.github/workflows/report-merged-pr.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml index 8ad83b34d36d1..a3a4ecbf609e5 100644 --- a/.github/workflows/serverless-benchmarks.yml +++ b/.github/workflows/serverless-benchmarks.yml @@ -24,7 +24,7 @@ jobs: sha: ${{ steps.prepare.outputs.sha }} steps: - name: Checkout ${{ github.base_ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -63,7 +63,7 @@ jobs: steps: - name: Checkout ${{ github.ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.sha }} persist-credentials: false diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml index 7be692d81d51a..fda6772398b9e 100644 --- a/.github/workflows/serverless-binary-size.yml +++ b/.github/workflows/serverless-binary-size.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write # Add comment to PR steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false @@ -36,7 +36,7 @@ jobs: fi - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml index ace5e88fbda98..b321f2fb82182 100644 --- a/.github/workflows/serverless-integration.yml +++ b/.github/workflows/serverless-integration.yml @@ -26,7 +26,7 @@ jobs: name: ${{ matrix.suite }} on ${{ matrix.architecture }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false @@ -40,7 +40,7 @@ jobs: run: sudo yarn global add serverless@^3.36.0 --prefix /usr/local - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension From 8949577ed72baea2dd7697db60ad37b838d3999e Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Wed, 18 Dec 2024 11:56:21 -0500 Subject: [PATCH 05/26] APM: Skip flaky trace config hostname test (#32233) --- comp/trace/config/config_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/comp/trace/config/config_test.go b/comp/trace/config/config_test.go index 9c9d45eb284ac..9706e2c784f4c 100644 --- a/comp/trace/config/config_test.go +++ b/comp/trace/config/config_test.go @@ -9,6 +9,7 @@ import ( "bufio" "bytes" "context" + _ "embed" "encoding/json" "errors" "net/http" @@ -245,6 +246,9 @@ func TestTelemetryEndpointsConfig(t *testing.T) { }) } +//go:embed testdata/stringcode.go.tmpl +var stringCodeBody string + func TestConfigHostname(t *testing.T) { t.Run("fail", func(t *testing.T) { overrides := map[string]interface{}{ @@ -360,11 +364,6 @@ func TestConfigHostname(t *testing.T) { }) t.Run("external", func(t *testing.T) { - body, err := os.ReadFile("testdata/stringcode.go.tmpl") - if err != nil { - t.Fatal(err) - } - // makeProgram creates a new binary file which returns the given response and exits to the OS // given the specified code, returning the path of the program. makeProgram := func(t *testing.T, response string, code int) string { @@ -372,7 +371,7 @@ func TestConfigHostname(t *testing.T) { if err != nil { t.Fatal(err) } - tmpl, err := template.New("program").Parse(string(body)) + tmpl, err := template.New("program").Parse(stringCodeBody) if err != nil { t.Fatal(err) } @@ -399,6 +398,7 @@ func TestConfigHostname(t *testing.T) { fallbackHostnameFunc = func() (string, error) { return "fallback.host", nil } t.Run("good", func(t *testing.T) { + t.Skip("Skip flaky test while we explore fixes.") bin := makeProgram(t, "host.name", 0) defer os.Remove(bin) From 623e2dcecf5174cf7519afe4913409ac739db9bf Mon Sep 17 00:00:00 2001 From: Romain Marcadier Date: Wed, 18 Dec 2024 18:16:00 +0100 Subject: [PATCH 06/26] [serverless/proxy]: override default ErrorHandler (#32326) --- pkg/serverless/proxy/proxy.go | 38 ++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/pkg/serverless/proxy/proxy.go b/pkg/serverless/proxy/proxy.go index 7d43dd36db257..1f8a2e3087658 100644 --- a/pkg/serverless/proxy/proxy.go +++ b/pkg/serverless/proxy/proxy.go @@ -8,6 +8,7 @@ package proxy import ( "context" + "errors" "net/http" "net/http/httputil" "net/url" @@ -68,9 +69,44 @@ func newProxy(target string, processor invocationlifecycle.InvocationProcessor) Scheme: "http", Host: target, } + proxy := httputil.NewSingleHostReverseProxy(url) + + // The default error handler logs "http: proxy error: %v" then returns an HTTP 502 (bad gateway) + // response. This is unfortunate because it lacks much any context on the original request that + // failed, and the commonly observed error today is "context deadline exceeded", which is not + // actionnable if you don't know what request it was for. It also logs to STDERR and does not + // honor the agent's log level. + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + log.Debugf( + "[serverless/proxy][%T] %s %s -- proxy error: %v", + // The dynamic type of processor informs about what kind of proxy this was (main/appsec) + processor, + // The request method and URL are useful to understand what exactly failed. We won't log + // the body (too large) or headers (risks containing sensitive data, such as API keys) + r.Method, r.URL, + // What happened that caused us to be called? + err, + ) + + // If the error is a [context.DeadlineExceeded], we return an HTTP 504 (gateway timeout) + // instead of the generic HTTP 502 (bad gateway) to give the client a better idea of what is + // going on (this may influence retry behavior, for example). + if errors.Is(err, context.DeadlineExceeded) { + w.WriteHeader(http.StatusGatewayTimeout) + } else { + // Return an HTTP 502 (bad gateway) error response; defer the retrying to the client. + w.WriteHeader(http.StatusBadGateway) + } + + // Writing the error message as best-effort, we simply debug-log any error that occur here. + if _, err := w.Write([]byte(err.Error())); err != nil { + log.Debugf("[serverless/proxy][%T] failed to write error message to response body: %v", processor, err) + } + } + return &runtimeProxy{ target: url, - proxy: httputil.NewSingleHostReverseProxy(url), + proxy: proxy, processor: processor, } } From 99d8d32638b9b302dc0566872b3df8d59e57d94d Mon Sep 17 00:00:00 2001 From: pducolin <45568537+pducolin@users.noreply.github.com> Date: Wed, 18 Dec 2024 18:16:10 +0100 Subject: [PATCH 07/26] [tasks] add back comments to test_infra_version.yaml (#32344) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Célian Raimbault <161456554+CelianR@users.noreply.github.com> --- tasks/buildimages.py | 16 +++++++++++++--- tasks/libs/ciproviders/gitlab_api.py | 5 ++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tasks/buildimages.py b/tasks/buildimages.py index e2bf5316e18c7..06ea84211ac1a 100644 --- a/tasks/buildimages.py +++ b/tasks/buildimages.py @@ -46,13 +46,23 @@ def update_test_infra_definitions(ctx: Context, commit_sha: str, go_mod_only: bo """ Update the test-infra-definition image version in the Gitlab CI as well as in the e2e go.mod """ - if not go_mod_only: - update_test_infra_def(".gitlab/common/test_infra_version.yml", commit_sha[:12], is_dev_image) - + print(f"Updating test-infra-definitions to {commit_sha}") with ctx.cd("test/new-e2e"): ctx.run(f"go get github.com/DataDog/test-infra-definitions@{commit_sha}") ctx.run("go mod tidy") + if not go_mod_only: + prefix_comment = """# File generated by inv buildimages.update-test-infra-definitions +# Please do not edit this file manually +# To update the test-infra-definitions version, run `inv buildimages.update-test-infra-definitions --commit-sha ` [--is-dev-image] +""" + update_test_infra_def( + file_path=".gitlab/common/test_infra_version.yml", + image_tag=commit_sha[:12], + is_dev_image=is_dev_image, + prefix_comment=prefix_comment, + ) + @task( help={ diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index adfd8aacfb569..f1c0b4784d2f6 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -1258,12 +1258,10 @@ def full_config_get_all_stages(full_config: dict) -> set[str]: return all_stages -def update_test_infra_def(file_path, image_tag, is_dev_image=False): +def update_test_infra_def(file_path, image_tag, is_dev_image=False, prefix_comment=""): """ Updates TEST_INFRA_DEFINITIONS_BUILDIMAGES in `.gitlab/common/test_infra_version.yml` file """ - import yaml - test_infra_def = {} with open(file_path) as test_infra_version_file: try: @@ -1276,6 +1274,7 @@ def update_test_infra_def(file_path, image_tag, is_dev_image=False): except yaml.YAMLError as e: raise Exit(f"Error while loading {file_path}: {e}") from e with open(file_path, "w") as test_infra_version_file: + test_infra_version_file.write(prefix_comment + ('\n\n' if prefix_comment else '')) # Add explicit_start=True to keep the document start marker --- # See "Document Start" in https://www.yaml.info/learn/document.html for more details yaml.dump(test_infra_def, test_infra_version_file, explicit_start=True) From 0d05fdd396532c8cbde7e43355681c6133a4b9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Wed, 18 Dec 2024 18:16:21 +0100 Subject: [PATCH 08/26] omnibus: python: probe the build environment for CC/CXX (#32340) --- omnibus/config/software/python3.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 2fe37a032fc24..1cd3c096363f2 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -54,11 +54,11 @@ # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default # compiler we use would end up being available in the system/docker image used by customers - if linux_target? && ENV["CC"] + if linux_target? && env["CC"] command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end - if linux_target? && ENV["CXX"] + if linux_target? && env["CXX"] command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end From 3b2649bfd399d2d93e3de64b91f23b05164d7df4 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Wed, 18 Dec 2024 18:22:22 +0100 Subject: [PATCH 09/26] [CWS] Allow specifying cgroup managers for dumps generation (#32287) --- pkg/config/setup/system_probe_cws.go | 1 + pkg/security/config/config.go | 4 ++ .../ebpf/c/include/constants/custom.h | 4 ++ .../ebpf/c/include/helpers/activity_dump.h | 22 ++++++---- pkg/security/ebpf/c/include/hooks/cgroup.h | 14 +++---- pkg/security/ebpf/c/include/maps.h | 2 +- pkg/security/probe/probe_ebpf.go | 8 +--- pkg/security/resolvers/cgroup/resolver.go | 18 +++++++- .../resolvers/process/resolver_ebpf.go | 2 +- pkg/security/resolvers/resolvers_ebpf.go | 4 ++ pkg/security/secl/containerutils/cgroup.go | 6 +++ pkg/security/secl/containerutils/helpers.go | 33 ++++++++------- .../secl/containerutils/helpers_test.go | 2 +- .../security_profile/dump/load_controller.go | 42 ++++++++++++++++--- pkg/security/security_profile/dump/manager.go | 21 ++++++++-- 15 files changed, 133 insertions(+), 50 deletions(-) diff --git a/pkg/config/setup/system_probe_cws.go b/pkg/config/setup/system_probe_cws.go index d2cc6276907d3..c689e1fd68e28 100644 --- a/pkg/config/setup/system_probe_cws.go +++ b/pkg/config/setup/system_probe_cws.go @@ -57,6 +57,7 @@ func initCWSSystemProbeConfig(cfg pkgconfigmodel.Config) { cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.min_timeout", "10m") cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.max_dump_size", 1750) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_cgroups_count", 5) + cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.cgroup_managers", []string{"docker", "podman", "containerd", "cri-o"}) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_event_types", []string{"exec", "open", "dns", "imds"}) cfg.BindEnv("runtime_security_config.activity_dump.cgroup_dump_timeout") // deprecated in favor of dump_duration cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.dump_duration", "900s") diff --git a/pkg/security/config/config.go b/pkg/security/config/config.go index ab6b0b6b591e1..9d165a4ae1900 100644 --- a/pkg/security/config/config.go +++ b/pkg/security/config/config.go @@ -101,6 +101,9 @@ type RuntimeSecurityConfig struct { // ActivityDumpTracedCgroupsCount defines the maximum count of cgroups that should be monitored concurrently. Leave this parameter to 0 to prevent the generation // of activity dumps based on cgroups. ActivityDumpTracedCgroupsCount int + // ActivityDumpCgroupsManagers defines the cgroup managers we generate dumps for. + ActivityDumpCgroupsManagers []string + // ActivityDumpTracedEventTypes defines the list of events that should be captured in an activity dump. Leave this // parameter empty to monitor all event types. If not already present, the `exec` event will automatically be added // to this list. @@ -368,6 +371,7 @@ func NewRuntimeSecurityConfig() (*RuntimeSecurityConfig, error) { ActivityDumpLoadControlPeriod: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.load_controller_period"), ActivityDumpLoadControlMinDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.min_timeout"), ActivityDumpTracedCgroupsCount: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.traced_cgroups_count"), + ActivityDumpCgroupsManagers: pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.cgroup_managers"), ActivityDumpTracedEventTypes: parseEventTypeStringSlice(pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.traced_event_types")), ActivityDumpCgroupDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.dump_duration"), ActivityDumpRateLimiter: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.rate_limiter"), diff --git a/pkg/security/ebpf/c/include/constants/custom.h b/pkg/security/ebpf/c/include/constants/custom.h index 88be17fa3c80b..1e2311bf8fc7e 100644 --- a/pkg/security/ebpf/c/include/constants/custom.h +++ b/pkg/security/ebpf/c/include/constants/custom.h @@ -196,4 +196,8 @@ static __attribute__((always_inline)) u64 get_imds_ip() { #define CGROUP_MANAGER_CRI 4 #define CGROUP_MANAGER_SYSTEMD 5 +#define CGROUP_MANAGER_MASK 0b111 +#define CGROUP_SYSTEMD_SERVICE (0 << 8) +#define CGROUP_SYSTEMD_SCOPE (1 << 8) + #endif diff --git a/pkg/security/ebpf/c/include/helpers/activity_dump.h b/pkg/security/ebpf/c/include/helpers/activity_dump.h index c0f8d246006bf..70e776cca8161 100644 --- a/pkg/security/ebpf/c/include/helpers/activity_dump.h +++ b/pkg/security/ebpf/c/include/helpers/activity_dump.h @@ -53,10 +53,16 @@ __attribute__((always_inline)) struct cgroup_tracing_event_t *get_cgroup_tracing return evt; } +__attribute__((always_inline)) u32 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { + u32 cgroup_manager = cgroup->cgroup_flags & CGROUP_MANAGER_MASK; + u32 supported = (cgroup->cgroup_flags != 0) && (bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_manager) != NULL); + return supported; +} + __attribute__((always_inline)) bool reserve_traced_cgroup_spot(struct cgroup_context_t *cgroup, u64 now, u64 cookie, struct activity_dump_config *config) { // insert dump config defaults - u32 defaults_key = 0; - struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &defaults_key); + u32 cgroup_flags = cgroup->cgroup_flags; + struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_flags); if (defaults == NULL) { // should never happen, ignore return false; @@ -102,11 +108,15 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return 0; } - if ((container->cgroup_context.cgroup_flags & 0b111) == CGROUP_MANAGER_SYSTEMD) { + if (!is_cgroup_activity_dumps_supported(&container->cgroup_context)) { return 0; } - copy_container_id(container->container_id, evt->container.container_id); + if ((container->cgroup_context.cgroup_flags&CGROUP_MANAGER_MASK) != CGROUP_MANAGER_SYSTEMD) { + copy_container_id(container->container_id, evt->container.container_id); + } else { + evt->container.container_id[0] = '\0'; + } evt->container.cgroup_context = container->cgroup_context; evt->cookie = cookie; evt->config = config; @@ -115,10 +125,6 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return cookie; } -__attribute__((always_inline)) u64 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { - return (cgroup->cgroup_flags != 0) && ((cgroup->cgroup_flags&0b111) != CGROUP_MANAGER_SYSTEMD); -} - __attribute__((always_inline)) u64 should_trace_new_process_cgroup(void *ctx, u64 now, u32 pid, struct container_context_t *container) { // should we start tracing this cgroup ? struct cgroup_context_t cgroup_context; diff --git a/pkg/security/ebpf/c/include/hooks/cgroup.h b/pkg/security/ebpf/c/include/hooks/cgroup.h index a231142b8f90a..b7ce66a870b92 100644 --- a/pkg/security/ebpf/c/include/hooks/cgroup.h +++ b/pkg/security/ebpf/c/include/hooks/cgroup.h @@ -189,13 +189,13 @@ static __attribute__((always_inline)) int trace__cgroup_write(ctx_t *ctx) { #endif int length = bpf_probe_read_str(prefix, sizeof(cgroup_prefix_t), container_id) & 0xff; - if (cgroup_flags == 0 && ( - (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') - || - (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') - )) { - cgroup_flags = CGROUP_MANAGER_SYSTEMD; - } else if (cgroup_flags != 0) { + if (cgroup_flags == 0) { + if (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SERVICE; + } else if (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SCOPE; + } + } else { bpf_probe_read(&new_entry.container.container_id, sizeof(new_entry.container.container_id), container_id); } diff --git a/pkg/security/ebpf/c/include/maps.h b/pkg/security/ebpf/c/include/maps.h index d6ab3ceb74dfb..3b821b19fe5cf 100644 --- a/pkg/security/ebpf/c/include/maps.h +++ b/pkg/security/ebpf/c/include/maps.h @@ -28,7 +28,7 @@ BPF_ARRAY_MAP(syscall_ctx_gen_id, u32, 1) BPF_ARRAY_MAP(syscall_ctx, char[MAX_SYSCALL_CTX_SIZE], MAX_SYSCALL_CTX_ENTRIES) BPF_HASH_MAP(activity_dumps_config, u64, struct activity_dump_config, 1) // max entries will be overridden at runtime -BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 1) +BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 5) BPF_HASH_MAP(traced_cgroups, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(cgroup_wait_list, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(traced_pids, u32, u64, 8192) // max entries will be overridden at runtime diff --git a/pkg/security/probe/probe_ebpf.go b/pkg/security/probe/probe_ebpf.go index f06cbe8fa45ae..5fd26b4f63f8a 100644 --- a/pkg/security/probe/probe_ebpf.go +++ b/pkg/security/probe/probe_ebpf.go @@ -818,15 +818,11 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)); err != nil { + cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)) + if err != nil { seclog.Debugf("Failed to resolve cgroup: %s", err) } else { event.CgroupTracing.CGroupContext = *cgroupContext - if cgroupContext.CGroupFlags.IsContainer() { - containerID, _ := containerutils.FindContainerID(cgroupContext.CGroupID) - event.CgroupTracing.ContainerContext.ContainerID = containerID - } - p.profileManagers.activityDumpManager.HandleCGroupTracingEvent(&event.CgroupTracing) } diff --git a/pkg/security/resolvers/cgroup/resolver.go b/pkg/security/resolvers/cgroup/resolver.go index 2137cccf14506..dcb1de1e6d378 100644 --- a/pkg/security/resolvers/cgroup/resolver.go +++ b/pkg/security/resolvers/cgroup/resolver.go @@ -48,6 +48,7 @@ type ResolverInterface interface { type Resolver struct { *utils.Notifier[Event, *cgroupModel.CacheEntry] sync.Mutex + cgroups *simplelru.LRU[model.PathKey, *model.CGroupContext] hostWorkloads *simplelru.LRU[containerutils.CGroupID, *cgroupModel.CacheEntry] containerWorkloads *simplelru.LRU[containerutils.ContainerID, *cgroupModel.CacheEntry] } @@ -80,6 +81,11 @@ func NewResolver() (*Resolver, error) { return nil, err } + cr.cgroups, err = simplelru.NewLRU(2048, func(_ model.PathKey, _ *model.CGroupContext) {}) + if err != nil { + return nil, err + } + return cr, nil } @@ -121,10 +127,19 @@ func (cr *Resolver) AddPID(process *model.ProcessCacheEntry) { } else { cr.hostWorkloads.Add(process.CGroup.CGroupID, newCGroup) } + cr.cgroups.Add(process.CGroup.CGroupFile, &process.CGroup) cr.NotifyListeners(CGroupCreated, newCGroup) } +// GetCGroupContext returns the cgroup context with the specified path key +func (cr *Resolver) GetCGroupContext(cgroupPath model.PathKey) (*model.CGroupContext, bool) { + cr.Lock() + defer cr.Unlock() + + return cr.cgroups.Get(cgroupPath) +} + // GetWorkload returns the workload referenced by the provided ID func (cr *Resolver) GetWorkload(id containerutils.ContainerID) (*cgroupModel.CacheEntry, bool) { if id == "" { @@ -171,6 +186,7 @@ func (cr *Resolver) deleteWorkloadPID(pid uint32, workload *cgroupModel.CacheEnt // check if the workload should be deleted if len(workload.PIDs) <= 0 { + cr.cgroups.Remove(workload.CGroupFile) cr.hostWorkloads.Remove(workload.CGroupID) if workload.ContainerID != "" { cr.containerWorkloads.Remove(workload.ContainerID) @@ -183,5 +199,5 @@ func (cr *Resolver) Len() int { cr.Lock() defer cr.Unlock() - return cr.hostWorkloads.Len() + cr.containerWorkloads.Len() + return cr.cgroups.Len() } diff --git a/pkg/security/resolvers/process/resolver_ebpf.go b/pkg/security/resolvers/process/resolver_ebpf.go index 5bcccdb52c540..409ab86fc2da7 100644 --- a/pkg/security/resolvers/process/resolver_ebpf.go +++ b/pkg/security/resolvers/process/resolver_ebpf.go @@ -340,7 +340,7 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc // Retrieve the container ID of the process from /proc containerID, cgroup, err := p.containerResolver.GetContainerContext(pid) if err != nil { - return fmt.Errorf("snapshot failed for %d: couldn't parse container ID: %w", proc.Pid, err) + return fmt.Errorf("snapshot failed for %d: couldn't parse container and cgroup context: %w", proc.Pid, err) } entry.ContainerID = containerID diff --git a/pkg/security/resolvers/resolvers_ebpf.go b/pkg/security/resolvers/resolvers_ebpf.go index b8899bbd99adf..c9adeb82e557b 100644 --- a/pkg/security/resolvers/resolvers_ebpf.go +++ b/pkg/security/resolvers/resolvers_ebpf.go @@ -219,6 +219,10 @@ func (r *EBPFResolvers) Start(ctx context.Context) error { // ResolveCGroupContext resolves the cgroup context from a cgroup path key func (r *EBPFResolvers) ResolveCGroupContext(pathKey model.PathKey, cgroupFlags containerutils.CGroupFlags) (*model.CGroupContext, error) { + if cgroupContext, found := r.CGroupResolver.GetCGroupContext(pathKey); found { + return cgroupContext, nil + } + path, err := r.DentryResolver.Resolve(pathKey, true) if err != nil { return nil, fmt.Errorf("failed to resolve cgroup file %v: %w", pathKey, err) diff --git a/pkg/security/secl/containerutils/cgroup.go b/pkg/security/secl/containerutils/cgroup.go index 74e7a64540c84..9cc6cca12d884 100644 --- a/pkg/security/secl/containerutils/cgroup.go +++ b/pkg/security/secl/containerutils/cgroup.go @@ -24,6 +24,12 @@ const ( CGroupManagerSystemd // systemd ) +// CGroup flags +const ( + SystemdService CGroupFlags = (0 << 8) + SystemdScope CGroupFlags = (1 << 8) +) + const ( // ContainerRuntimeDocker is used to specify that a container is managed by Docker ContainerRuntimeDocker = "docker" diff --git a/pkg/security/secl/containerutils/helpers.go b/pkg/security/secl/containerutils/helpers.go index 0e46e87af7f2a..311322f94ffc6 100644 --- a/pkg/security/secl/containerutils/helpers.go +++ b/pkg/security/secl/containerutils/helpers.go @@ -29,39 +29,42 @@ func init() { containerIDPattern = regexp.MustCompile(ContainerIDPatternStr) } -func isSystemdCgroup(cgroup CGroupID) bool { - return strings.HasSuffix(string(cgroup), ".service") || strings.HasSuffix(string(cgroup), ".scope") +func isSystemdScope(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".scope") +} + +func isSystemdService(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".service") +} + +func getSystemdCGroupFlags(cgroup CGroupID) uint64 { + if isSystemdScope(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdScope) + } else if isSystemdService(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdService) + } + return 0 } // FindContainerID extracts the first sub string that matches the pattern of a container ID along with the container flags induced from the container runtime prefix func FindContainerID(s CGroupID) (ContainerID, uint64) { match := containerIDPattern.FindIndex([]byte(s)) if match == nil { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - - return "", 0 + return "", getSystemdCGroupFlags(s) } // first, check what's before if match[0] != 0 { previousChar := string(s[match[0]-1]) if strings.ContainsAny(previousChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } // then, check what's after if match[1] < len(s) { nextChar := string(s[match[1]]) if strings.ContainsAny(nextChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } diff --git a/pkg/security/secl/containerutils/helpers_test.go b/pkg/security/secl/containerutils/helpers_test.go index b5474b0df5016..5a2066e0295f2 100644 --- a/pkg/security/secl/containerutils/helpers_test.go +++ b/pkg/security/secl/containerutils/helpers_test.go @@ -63,7 +63,7 @@ func TestFindContainerID(t *testing.T) { { // Some random path which could match garden format input: "/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-f9176c6a-2a34-4ce2-86af-60d16888ed8e.scope", output: "", - flags: CGroupManagerSystemd, + flags: CGroupManagerSystemd | CGroupManager(SystemdScope), }, { // GARDEN with prefix / suffix input: "prefix01234567-0123-4567-890a-bcdesuffix", diff --git a/pkg/security/security_profile/dump/load_controller.go b/pkg/security/security_profile/dump/load_controller.go index ae8767bcfd153..f9bc80cdd3b4f 100644 --- a/pkg/security/security_profile/dump/load_controller.go +++ b/pkg/security/security_profile/dump/load_controller.go @@ -16,6 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/metrics" + "github.com/DataDog/datadog-agent/pkg/security/secl/containerutils" "github.com/DataDog/datadog-agent/pkg/security/secl/model" "github.com/DataDog/datadog-agent/pkg/security/seclog" ) @@ -34,6 +35,7 @@ type ActivityDumpLoadController struct { // eBPF maps activityDumpConfigDefaults *ebpf.Map + activityDumpLoadConfig map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig } // NewActivityDumpLoadController returns a new activity dump load controller @@ -58,7 +60,11 @@ func NewActivityDumpLoadController(adm *ActivityDumpManager) (*ActivityDumpLoadC }, nil } -func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDumpLoadConfig { +func (lc *ActivityDumpLoadController) getDefaultLoadConfigs() (map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig, error) { + if lc.activityDumpLoadConfig != nil { + return lc.activityDumpLoadConfig, nil + } + defaults := NewActivityDumpLoadConfig( lc.adm.config.RuntimeSecurity.ActivityDumpTracedEventTypes, lc.adm.config.RuntimeSecurity.ActivityDumpCgroupDumpTimeout, @@ -68,14 +74,38 @@ func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDump lc.adm.resolvers.TimeResolver, ) defaults.WaitListTimestampRaw = uint64(lc.adm.config.RuntimeSecurity.ActivityDumpCgroupWaitListTimeout) - return defaults + + allDefaultConfigs := map[string]containerutils.CGroupManager{ + containerutils.CGroupManagerDocker.String(): containerutils.CGroupManagerDocker, + containerutils.CGroupManagerPodman.String(): containerutils.CGroupManagerPodman, + containerutils.CGroupManagerCRI.String(): containerutils.CGroupManagerCRI, + containerutils.CGroupManagerCRIO.String(): containerutils.CGroupManagerCRIO, + containerutils.CGroupManagerSystemd.String(): containerutils.CGroupManagerSystemd, + } + defaultConfigs := make(map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig) + for _, cgroupManager := range lc.adm.config.RuntimeSecurity.ActivityDumpCgroupsManagers { + cgroupManager, found := allDefaultConfigs[cgroupManager] + if !found { + return nil, fmt.Errorf("unsupported cgroup manager '%s'", cgroupManager) + } + defaultConfigs[cgroupManager] = defaults + } + lc.activityDumpLoadConfig = defaultConfigs + return defaultConfigs, nil } -// PushCurrentConfig pushes the current load controller config to kernel space -func (lc *ActivityDumpLoadController) PushCurrentConfig() error { +// PushDefaultCurrentConfigs pushes the current load controller configs to kernel space +func (lc *ActivityDumpLoadController) PushDefaultCurrentConfigs() error { + defaultConfigs, err := lc.getDefaultLoadConfigs() + if err != nil { + return err + } + // push default load config values - if err := lc.activityDumpConfigDefaults.Put(uint32(0), lc.getDefaultLoadConfig()); err != nil { - return fmt.Errorf("couldn't update default activity dump load config: %w", err) + for cgroupManager, defaultConfig := range defaultConfigs { + if err := lc.activityDumpConfigDefaults.Put(uint32(cgroupManager), defaultConfig); err != nil { + return fmt.Errorf("couldn't update default activity dump load config: %w", err) + } } return nil } diff --git a/pkg/security/security_profile/dump/manager.go b/pkg/security/security_profile/dump/manager.go index 079f45c13bc21..be81a28ca5c4d 100644 --- a/pkg/security/security_profile/dump/manager.go +++ b/pkg/security/security_profile/dump/manager.go @@ -321,7 +321,8 @@ func NewActivityDumpManager(config *config.Config, statsdClient statsd.ClientInt if err != nil { return nil, fmt.Errorf("couldn't instantiate the activity dump load controller: %w", err) } - if err = loadController.PushCurrentConfig(); err != nil { + + if err = loadController.PushDefaultCurrentConfigs(); err != nil { return nil, fmt.Errorf("failed to push load controller config settings to kernel space: %w", err) } adm.loadController = loadController @@ -449,8 +450,8 @@ func (adm *ActivityDumpManager) HandleCGroupTracingEvent(event *model.CgroupTrac adm.Lock() defer adm.Unlock() - if len(event.ContainerContext.ContainerID) == 0 { - seclog.Warnf("received a cgroup tracing event with an empty container ID") + if len(event.CGroupContext.CGroupID) == 0 { + seclog.Warnf("received a cgroup tracing event with an empty cgroup ID") return } @@ -514,7 +515,19 @@ workloadLoop: } // if we're still here, we can start tracing this workload - if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *adm.loadController.getDefaultLoadConfig()); err != nil { + defaultConfigs, err := adm.loadController.getDefaultLoadConfigs() + if err != nil { + seclog.Errorf("%v", err) + continue + } + + defaultConfig, found := defaultConfigs[containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags)] + if !found { + seclog.Errorf("Failed to find default activity dump config for %s", containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags).String()) + continue + } + + if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *defaultConfig); err != nil { if !errors.Is(err, unix.E2BIG) { seclog.Debugf("%v", err) break From 83c3dbd9266341686f223b4c336f41b12b9f9f79 Mon Sep 17 00:00:00 2001 From: "agent-platform-auto-pr[bot]" <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:25:41 +0000 Subject: [PATCH 10/26] [test-infra-definitions][automated] Bump test-infra-definitions to 221bbc806266eb15b90cb875deb79180e7591fbc (#32345) Co-authored-by: agent-platform-auto-pr[bot] <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> --- .gitlab/common/test_infra_version.yml | 7 ++----- test/new-e2e/go.mod | 12 ++++++------ test/new-e2e/go.sum | 24 ++++++++++++------------ 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index 9008ad3e857b1..944343f20da2f 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -1,7 +1,4 @@ --- variables: - # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev - # and check the job creating the image to make sure you have the right SHA prefix - TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" - # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 6459608ed9fa + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 221bbc806266 + TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: '' diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 92faae31be87d..be219da1e3d81 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -58,7 +58,7 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa + github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 @@ -72,10 +72,10 @@ require ( github.com/kr/pretty v0.3.1 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c github.com/pkg/sftp v1.13.7 - github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 - github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 - github.com/pulumi/pulumi/sdk/v3 v3.140.0 + github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 + github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 + github.com/pulumi/pulumi/sdk/v3 v3.142.0 github.com/samber/lo v1.47.0 github.com/stretchr/testify v1.10.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -171,7 +171,7 @@ require ( github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/hcl/v2 v2.22.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index e922c7c7a56b9..7284ab1424a36 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -17,8 +17,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa h1:l8KLWgU9l2qTlMtu4ing3V6PptTO+suaU8zusc45IiM= -github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa/go.mod h1:1PAUwGjC25ACjfft4HrLEmHliuajlvjzcLFWpuqAIyk= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 h1:w+uoG7RdtPiW6YvJvWrzWOC78eNzCemcH8ZM6tJoUBw= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266/go.mod h1:+13pRFKChJo9VZ0WcHFm5GTPCNYyim3hxFjy6cpcLG8= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= @@ -268,8 +268,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M= +github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -403,10 +403,10 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435 github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= github.com/pulumi/esc v0.10.0 h1:jzBKzkLVW0mePeanDRfqSQoCJ5yrkux0jIwAkUxpRKE= github.com/pulumi/esc v0.10.0/go.mod h1:2Bfa+FWj/xl8CKqRTWbWgDX0SOD4opdQgvYSURTGK2c= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 h1:wA38Ep4sEphX+3YGwFfaxRHs7NQv8dNObFepX6jaRa4= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1/go.mod h1:m/ejZ2INurqq/ncDjJfgC1Ff/lnbt0J/uO33BnPVots= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 h1:6082hB+ILpPB/0V5F+LTmHbX1BO54tCVOQCVOL/FYI4= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1/go.mod h1:z2bnBPHNYfk72IW1P01H9qikBtBSBhCwi3QpH6Y/38Q= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 h1:OvCLqUueOja9YE2WEGPYAw+lKHFRbLQ7QjwX55+uNsA= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0/go.mod h1:FFzye44v9E0BgaFXVB/9X7KH0S0MapoXEy2YonrQfz4= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 h1:jil2EBzZnKsRDrLfvx2gnAaq17HQLrTbpPsIb3h+98U= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0/go.mod h1:r+K4M7jnLqvvQDeR/0mBRq2EPZaqsDg24Ciy3ml/thA= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1 h1:miIJy4njnFYw7VxMLvEztoMPr9zYC2kqBTwRlaFAf48= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1/go.mod h1:LR1QBq0C1NIhmD9E0uKozCAu32j5qsamhrIsTSNVMS8= github.com/pulumi/pulumi-azure-native-sdk/compute/v2 v2.73.1 h1:79HTKSE1uJQolCRUHRFnIbSPNSIhxekIhznHnjpLi6s= @@ -427,16 +427,16 @@ github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 h1:s2Cpu6E2lmADNUbutbJGm6O+O9j0mBLlrh github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 h1:21oSj+TKlKTzQcxN9Hik7iSNNHPUQXN4s3itOnahy/w= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0/go.mod h1:YaEZms1NgXFqGhObKVofcAeWXu2V+3t/BAXdHQZq7fU= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 h1:quqoGsLbF7lpGpGU4mi5WfVLIAo4gfvoQeYYmemx1Dg= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3/go.mod h1:9dBA6+rtpKmyZB3k1XryUOHDOuNdoTODFKEEZZCtrz8= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 h1:7AjJpUyW6YHHpZr0bI6Fy1A3/b7ERxq1LAo5mlyNN1Y= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0/go.mod h1:ATS+UN8pguMxypQAK+SaPewesU+UN5dpf93PNqVuHzs= github.com/pulumi/pulumi-libvirt/sdk v0.5.3 h1:CiUGTweLLIxbAbADxxnwPv4BK8pxXfU8urokJvK1ihM= github.com/pulumi/pulumi-libvirt/sdk v0.5.3/go.mod h1:gAhyIZKtzs4rknrl8fu8BQnyqijAmViFbaUkyuHt4xY= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 h1:39rhOe/PTUGMYia8pR5T2wbxxMt2pwrlonf0ncYKSzE= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7/go.mod h1:cxxDhJzUPt/YElfvlWa15Q4NGF6XXS8kUs4OQsCxSBk= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 h1:tXemWrzeVTqG8zq6hBdv1TdPFXjgZ+dob63a/6GlF1o= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1/go.mod h1:hODo3iEmmXDFOXqPK+V+vwI0a3Ww7BLjs5Tgamp86Ng= -github.com/pulumi/pulumi/sdk/v3 v3.140.0 h1:+Z/RBvdYg7tBNkBwk4p/FzlV7niBT3TbLAICq/Y0LDU= -github.com/pulumi/pulumi/sdk/v3 v3.140.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= +github.com/pulumi/pulumi/sdk/v3 v3.142.0 h1:SmcVddGuvwAh3g3XUVQQ5gVRQUKH1yZ6iETpDNHIHlw= +github.com/pulumi/pulumi/sdk/v3 v3.142.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= github.com/pulumiverse/pulumi-time/sdk v0.1.0 h1:xfi9HKDgV+GgDxQ23oSv9KxC3DQqViGTcMrJICRgJv0= github.com/pulumiverse/pulumi-time/sdk v0.1.0/go.mod h1:NUa1zA74DF002WrM6iF111A6UjX9knPpXufVRvBwNyg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= From c29edf5b7aaad704dd85a89e72f42efee9e7556d Mon Sep 17 00:00:00 2001 From: Wassim Dhif Date: Wed, 18 Dec 2024 18:50:32 +0100 Subject: [PATCH 11/26] feat(apm): implement external data resolution (#32295) Signed-off-by: Wassim DHIF --- cmd/trace-agent/config/remote/config.go | 2 +- comp/otelcol/ddflareextension/impl/go.mod | 2 + .../exporter/datadogexporter/go.mod | 2 + .../otlp/components/statsprocessor/go.mod | 2 + comp/trace/config/setup.go | 6 ++- go.mod | 2 +- modules.yml | 3 +- pkg/proto/pbgo/core/model.pb.go | 4 +- pkg/trace/api/api.go | 2 +- pkg/trace/api/api_test.go | 5 ++- pkg/trace/api/container.go | 3 +- pkg/trace/api/container_linux.go | 43 ++++++++++++++++--- pkg/trace/api/debugger.go | 2 +- pkg/trace/api/evp_proxy.go | 2 +- pkg/trace/api/internal/header/headers.go | 10 +++++ pkg/trace/api/otlp.go | 4 +- pkg/trace/api/pipeline_stats.go | 2 +- pkg/trace/api/profiles.go | 2 +- pkg/trace/api/symdb.go | 2 +- pkg/trace/config/config.go | 4 ++ pkg/trace/go.mod | 2 + pkg/trace/stats/oteltest/go.mod | 2 + .../external_data_apm-6538531e3f34f305.yaml | 13 ++++++ test/otel/go.mod | 2 + 24 files changed, 100 insertions(+), 23 deletions(-) create mode 100644 releasenotes/notes/external_data_apm-6538531e3f34f305.yaml diff --git a/cmd/trace-agent/config/remote/config.go b/cmd/trace-agent/config/remote/config.go index d2c511a4586e7..6b650364743a7 100644 --- a/cmd/trace-agent/config/remote/config.go +++ b/cmd/trace-agent/config/remote/config.go @@ -43,7 +43,7 @@ func putBuffer(buffer *bytes.Buffer) { // ConfigHandler is the HTTP handler for configs func ConfigHandler(r *api.HTTPReceiver, cf rcclient.ConfigFetcher, cfg *config.AgentConfig, statsd statsd.ClientInterface, timing timing.Reporter) http.Handler { - cidProvider := api.NewIDProvider(cfg.ContainerProcRoot) + cidProvider := api.NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo) return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer timing.Since("datadog.trace_agent.receiver.config_process_ms", time.Now()) tags := r.TagStats(api.V07, req.Header, "").AsTags() diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 7af5d796d92fb..21d3f17b9066a 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -13,6 +13,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/tagger/tags => ../../../core/tagger/tags github.com/DataDog/datadog-agent/comp/core/tagger/types => ../../../core/tagger/types github.com/DataDog/datadog-agent/comp/core/tagger/utils => ../../../core/tagger/utils @@ -151,6 +152,7 @@ require ( require go.opentelemetry.io/collector/extension/extensiontest v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index c3e28a412d2e6..301ef0f3b1f48 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../../../core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../../../../def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../../../../forwarder/defaultforwarder @@ -138,6 +139,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/status v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 4ad268c10dfcb..18606500a04d2 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -3,6 +3,7 @@ module github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsproces go 1.22.0 replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../metricsclient github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../../../../comp/trace/compression/impl-gzip @@ -33,6 +34,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.3 // indirect diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index f19c5eeaf8a4f..06a157dc6418b 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -19,13 +19,14 @@ import ( "go.opentelemetry.io/collector/component/componenttest" - apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" corecompcfg "github.com/DataDog/datadog-agent/comp/core/config" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/configcheck" + apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/env" "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -121,6 +122,9 @@ func prepareConfig(c corecompcfg.Component, tagger tagger.Component) (*config.Ag cfg.ContainerTags = func(cid string) ([]string, error) { return tagger.Tag(types.NewEntityID(types.ContainerID, cid), types.HighCardinality) } + cfg.ContainerIDFromOriginInfo = func(originInfo origindetection.OriginInfo) (string, error) { + return tagger.GenerateContainerIDFromOriginInfo(originInfo) + } cfg.ContainerProcRoot = coreConfigObject.GetString("container_proc_root") cfg.GetAgentAuthToken = apiutil.GetAuthToken return cfg, nil diff --git a/go.mod b/go.mod index 30981dbec9ce2..c8de7db9cc5f0 100644 --- a/go.mod +++ b/go.mod @@ -654,7 +654,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 github.com/DataDog/datadog-agent/comp/core/status v0.59.0-rc.6 github.com/DataDog/datadog-agent/comp/core/status/statusimpl v0.56.0-rc.3 - github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-00010101000000-000000000000 + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/DataDog/datadog-agent/comp/core/tagger/tags v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/core/tagger/types v0.59.0 github.com/DataDog/datadog-agent/comp/core/telemetry v0.59.0 diff --git a/modules.yml b/modules.yml index 291db46bc53cc..5d5e5cb2057ff 100644 --- a/modules.yml +++ b/modules.yml @@ -35,7 +35,8 @@ modules: comp/core/status: used_by_otel: true comp/core/status/statusimpl: default - comp/core/tagger/origindetection: default + comp/core/tagger/origindetection: + used_by_otel: true comp/core/tagger/tags: used_by_otel: true comp/core/tagger/types: diff --git a/pkg/proto/pbgo/core/model.pb.go b/pkg/proto/pbgo/core/model.pb.go index 016a2cf53e260..9262b4dc43a5a 100644 --- a/pkg/proto/pbgo/core/model.pb.go +++ b/pkg/proto/pbgo/core/model.pb.go @@ -1155,8 +1155,8 @@ type GenerateContainerIDFromOriginInfoRequest_ExternalData struct { unknownFields protoimpl.UnknownFields Init *bool `protobuf:"varint,1,opt,name=init,proto3,oneof" json:"init,omitempty"` // Init is true if the container is an init container. - ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name as seen by the Admission Controller. - PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID as seen by the Admission Controller. + ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name in the Kubernetes Pod spec. + PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID in the Kubernetes Pod spec. } func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) Reset() { diff --git a/pkg/trace/api/api.go b/pkg/trace/api/api.go index c74ecbb137fb9..5cb57d2478a27 100644 --- a/pkg/trace/api/api.go +++ b/pkg/trace/api/api.go @@ -141,7 +141,7 @@ func NewHTTPReceiver( } } log.Infof("Receiver configured with %d decoders and a timeout of %dms", semcount, conf.DecoderTimeout) - containerIDProvider := NewIDProvider(conf.ContainerProcRoot) + containerIDProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) telemetryForwarder := NewTelemetryForwarder(conf, containerIDProvider, statsd) return &HTTPReceiver{ Stats: info.NewReceiverStats(), diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index 9a468dc164487..2a1bdafe71307 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/trace/config" @@ -576,7 +577,9 @@ func TestDecodeV05(t *testing.T) { req, err := http.NewRequest("POST", "/v0.5/traces", bytes.NewReader(b)) assert.NoError(err) req.Header.Set(header.ContainerID, "abcdef123789456") - tp, err := decodeTracerPayload(v05, req, NewIDProvider(""), "python", "3.8.1", "1.2.3") + tp, err := decodeTracerPayload(v05, req, NewIDProvider("", func(_ origindetection.OriginInfo) (string, error) { + return "abcdef123789456", nil + }), "python", "3.8.1", "1.2.3") assert.NoError(err) assert.EqualValues(tp, &pb.TracerPayload{ ContainerID: "abcdef123789456", diff --git a/pkg/trace/api/container.go b/pkg/trace/api/container.go index 508277c5abaf9..0c789f9d06ca7 100644 --- a/pkg/trace/api/container.go +++ b/pkg/trace/api/container.go @@ -12,6 +12,7 @@ import ( "net" "net/http" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" ) @@ -28,7 +29,7 @@ type IDProvider interface { type idProvider struct{} // NewIDProvider initializes an IDProvider instance, in non-linux environments the procRoot arg is unused. -func NewIDProvider(_ string) IDProvider { +func NewIDProvider(_ string, _ func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { return &idProvider{} } diff --git a/pkg/trace/api/container_linux.go b/pkg/trace/api/container_linux.go index 2129a6377c97f..6dd12d227894d 100644 --- a/pkg/trace/api/container_linux.go +++ b/pkg/trace/api/container_linux.go @@ -18,6 +18,7 @@ import ( "syscall" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/util/cgroups" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -96,7 +97,7 @@ func (i *noCgroupsProvider) GetContainerID(_ context.Context, h http.Header) str } // NewIDProvider initializes an IDProvider instance using the provided procRoot to perform cgroups lookups in linux environments. -func NewIDProvider(procRoot string) IDProvider { +func NewIDProvider(procRoot string, containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { // taken from pkg/util/containers/metrics/system.collector_linux.go var hostPrefix string if strings.HasPrefix(procRoot, "/host") { @@ -120,10 +121,11 @@ func NewIDProvider(procRoot string) IDProvider { } c := NewCache(1 * time.Minute) return &cgroupIDProvider{ - procRoot: procRoot, - controller: cgroupController, - cache: c, - reader: reader, + procRoot: procRoot, + controller: cgroupController, + cache: c, + reader: reader, + containerIDFromOriginInfo: containerIDFromOriginInfo, } } @@ -131,8 +133,9 @@ type cgroupIDProvider struct { procRoot string controller string // reader is used to retrieve the container ID from its cgroup v2 inode. - reader *cgroups.Reader - cache *Cache + reader *cgroups.Reader + cache *Cache + containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) } // GetContainerID returns the container ID. @@ -157,6 +160,11 @@ func (c *cgroupIDProvider) GetContainerID(ctx context.Context, h http.Header) st return containerID } + // Retrieve container ID from External Data header + if externalData := h.Get(header.ExternalData); externalData != "" { + return c.resolveContainerIDFromExternalData(externalData) + } + return "" } @@ -296,6 +304,27 @@ func (c *cgroupIDProvider) getCachedContainerID(key string, retrievalFunc func() return val, nil } +// resolveContainerIDFromExternalData returns the container ID for the given External Data. +func (c *cgroupIDProvider) resolveContainerIDFromExternalData(rawExternalData string) string { + var generatedContainerID string + + externalData, err := origindetection.ParseExternalData(rawExternalData) + if err != nil { + log.Errorf("Could not parse external data (%s): %v", rawExternalData, err) + return "" + } + generatedContainerID, err = c.containerIDFromOriginInfo(origindetection.OriginInfo{ + ExternalData: externalData, + ProductOrigin: origindetection.ProductOriginAPM, + }) + if err != nil { + log.Errorf("Could not generate container ID from external data (%s): %v", rawExternalData, err) + return "" + } + + return generatedContainerID +} + // The below cache is copied from /pkg/util/containers/v2/metrics/provider/cache.go. It is not // imported to avoid making the datadog-agent module a dependency of the pkg/trace module. The // datadog-agent module contains replace directives which are not inherited by packages that diff --git a/pkg/trace/api/debugger.go b/pkg/trace/api/debugger.go index 61164f1d3ed02..51816276a3f3b 100644 --- a/pkg/trace/api/debugger.go +++ b/pkg/trace/api/debugger.go @@ -81,7 +81,7 @@ func debuggerErrorHandler(err error) http.Handler { // newDebuggerProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newDebuggerProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/api/evp_proxy.go b/pkg/trace/api/evp_proxy.go index 452d73017aa8d..2795e6304b605 100644 --- a/pkg/trace/api/evp_proxy.go +++ b/pkg/trace/api/evp_proxy.go @@ -86,7 +86,7 @@ func evpProxyForwarder(conf *config.AgentConfig, statsd statsd.ClientInterface) req.Header["X-Forwarded-For"] = nil }, ErrorLog: logger, - Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot), statsd}, + Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo), statsd}, } } diff --git a/pkg/trace/api/internal/header/headers.go b/pkg/trace/api/internal/header/headers.go index 8ae458be834f5..88cb3549b2836 100644 --- a/pkg/trace/api/internal/header/headers.go +++ b/pkg/trace/api/internal/header/headers.go @@ -25,6 +25,16 @@ const ( // * "ci-,in-" LocalData = "Datadog-Entity-ID" + // ExternalData is a list that contain prefixed-items, split by a ','. Current items are: + // * "it-" if the container is an init container. + // * "cn-" for the container name. + // * "pu-" for the pod UID. + // Order does not matter. + // Possible values: + // * "it-false,cn-nginx,pu-3413883c-ac60-44ab-96e0-9e52e4e173e2" + // * "cn-init,pu-cb4aba1d-0129-44f1-9f1b-b4dc5d29a3b3,it-true" + ExternalData = "Datadog-External-Env" + // Lang specifies the name of the header which contains the language from // which the traces originate. Lang = "Datadog-Meta-Lang" diff --git a/pkg/trace/api/otlp.go b/pkg/trace/api/otlp.go index 88491ea9052ec..eb136d267e090 100644 --- a/pkg/trace/api/otlp.go +++ b/pkg/trace/api/otlp.go @@ -9,7 +9,6 @@ import ( "context" "encoding/hex" "fmt" - "github.com/DataDog/datadog-agent/pkg/trace/transform" "math" "net" "net/http" @@ -26,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/trace/sampler" "github.com/DataDog/datadog-agent/pkg/trace/timing" "github.com/DataDog/datadog-agent/pkg/trace/traceutil" + "github.com/DataDog/datadog-agent/pkg/trace/transform" "github.com/DataDog/datadog-go/v5/statsd" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" @@ -102,7 +102,7 @@ func NewOTLPReceiver(out chan<- *Payload, cfg *config.AgentConfig, statsd statsd enableReceiveResourceSpansV2Val = 1.0 } _ = statsd.Gauge("datadog.trace_agent.otlp.enable_receive_resource_spans_v2", enableReceiveResourceSpansV2Val, nil, 1) - return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} + return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} } // Start starts the OTLPReceiver, if any of the servers were configured as active. diff --git a/pkg/trace/api/pipeline_stats.go b/pkg/trace/api/pipeline_stats.go index b6fe3f2cd161c..b8daf12590bf7 100644 --- a/pkg/trace/api/pipeline_stats.go +++ b/pkg/trace/api/pipeline_stats.go @@ -71,7 +71,7 @@ func pipelineStatsErrorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. func newPipelineStatsProxy(conf *config.AgentConfig, urls []*url.URL, apiKeys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { log.Debug("[pipeline_stats] Creating reverse proxy") - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/profiles.go b/pkg/trace/api/profiles.go index 9b43efc94de74..19add3e307daa 100644 --- a/pkg/trace/api/profiles.go +++ b/pkg/trace/api/profiles.go @@ -109,7 +109,7 @@ func errorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. // For more details please see multiTransport. func newProfileProxy(conf *config.AgentConfig, targets []*url.URL, keys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/symdb.go b/pkg/trace/api/symdb.go index 31b96c494ce5f..b238b478eb65c 100644 --- a/pkg/trace/api/symdb.go +++ b/pkg/trace/api/symdb.go @@ -62,7 +62,7 @@ func symDBErrorHandler(err error) http.Handler { // newSymDBProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newSymDBProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getSymDBDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/config/config.go b/pkg/trace/config/config.go index 5d0e59f059e39..cbf0b0ea5c3a3 100644 --- a/pkg/trace/config/config.go +++ b/pkg/trace/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/obfuscate" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/trace/log" @@ -445,6 +446,9 @@ type AgentConfig struct { // ContainerTags ... ContainerTags func(cid string) ([]string, error) `json:"-"` + // ContainerIDFromOriginInfo ... + ContainerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) `json:"-"` + // ContainerProcRoot is the root dir for `proc` info ContainerProcRoot string diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 51564f328f09e..29bf241afd7b8 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -52,6 +52,7 @@ require ( ) require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/shirou/gopsutil/v4 v4.24.11 go.opentelemetry.io/collector/component/componenttest v0.115.0 ) @@ -115,6 +116,7 @@ require ( replace ( github.com/DataDog/datadog-agent => ../../ + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../comp/trace/compression/impl-gzip github.com/DataDog/datadog-agent/comp/trace/compression/impl-zstd => ../../comp/trace/compression/impl-zstd diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index c3759db9cc74b..071e7c2c2eba0 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -20,6 +20,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect @@ -87,6 +88,7 @@ require ( ) replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../../../../comp/otelcol/otlp/components/metricsclient github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ../../../../comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../comp/trace/compression/def diff --git a/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml new file mode 100644 index 0000000000000..4aa3e9ef23e70 --- /dev/null +++ b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Implement External Data resolution for APM. This is needed to support the + latest Origin Detection spec and resolution with nested virtualization. + diff --git a/test/otel/go.mod b/test/otel/go.mod index a4523c0b33b0b..88d5f3a127be5 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ./../../comp/core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ./../../comp/core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../comp/core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ./../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ./../../comp/def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../comp/forwarder/defaultforwarder @@ -129,6 +130,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect From e92df0ab7af2969d5945d75ab92b82fce34bd1e3 Mon Sep 17 00:00:00 2001 From: Raphael Gavache Date: Wed, 18 Dec 2024 19:33:19 +0100 Subject: [PATCH 12/26] [fleet] use custom tracer (#32301) --- LICENSE-3rdparty.csv | 3 - cmd/installer-downloader/main.go | 6 +- .../subcommands/installer/command.go | 12 +- .../subcommands/installer/umask_nix.go | 2 +- .../subcommands/installer/umask_windows.go | 2 +- .../agentcrashdetectimpl/agentcrashdetect.go | 13 +- .../telemetry/telemetryimpl/telemetry.go | 12 +- pkg/fleet/daemon/daemon.go | 2 +- pkg/fleet/installer/setup/common/setup.go | 4 +- pkg/fleet/internal/exec/installer_exec.go | 2 +- pkg/fleet/telemetry/http_wrapper.go | 84 +++++ pkg/fleet/telemetry/span.go | 138 +++++++- pkg/fleet/telemetry/telemetry.go | 312 ++++++------------ pkg/fleet/telemetry/telemetry_test.go | 222 +++++++++++++ pkg/fleet/telemetry/tracer.go | 92 ++++++ pkg/internaltelemetry/client.go | 24 +- pkg/internaltelemetry/traces.go | 41 +++ 17 files changed, 715 insertions(+), 256 deletions(-) create mode 100644 pkg/fleet/telemetry/http_wrapper.go create mode 100644 pkg/fleet/telemetry/telemetry_test.go create mode 100644 pkg/fleet/telemetry/tracer.go create mode 100644 pkg/internaltelemetry/traces.go diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 92e5c072e3c8e..13955c7e8a940 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2906,9 +2906,6 @@ core,google.golang.org/protobuf/types/known/timestamppb,BSD-3-Clause,Copyright ( core,google.golang.org/protobuf/types/known/wrapperspb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,google.golang.org/protobuf/types/pluginpb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,gopkg.in/DataDog/dd-trace-go.v1/appsec/events,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/datastreams/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext,Apache-2.0,"Copyright 2016-Present Datadog, Inc." diff --git a/cmd/installer-downloader/main.go b/cmd/installer-downloader/main.go index f7b67910ae94e..60e2141af944e 100644 --- a/cmd/installer-downloader/main.go +++ b/cmd/installer-downloader/main.go @@ -44,12 +44,12 @@ func main() { ctx := context.Background() t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor)) - _ = t.Start(ctx) - defer func() { _ = t.Stop(ctx) }() var err error span, ctx := telemetry.StartSpanFromEnv(ctx, fmt.Sprintf("downloader-%s", Flavor)) - defer func() { span.Finish(err) }() err = runDownloader(ctx, env, Version, Flavor) + + span.Finish(err) + t.Stop() if err != nil { fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err) os.Exit(1) diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 66ec6b1c74c57..f394d9955d407 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -87,7 +87,7 @@ func UnprivilegedCommands(_ *command.GlobalParams) []*cobra.Command { type cmd struct { t *telemetry.Telemetry ctx context.Context - span telemetry.Span + span *telemetry.Span env *env.Env } @@ -107,10 +107,7 @@ func newCmd(operation string) *cmd { func (c *cmd) Stop(err error) { c.span.Finish(err) if c.t != nil { - err := c.t.Stop(context.Background()) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to stop telemetry: %v\n", err) - } + c.t.Stop() } } @@ -225,11 +222,6 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { site = config.Site } t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands - err := t.Start(context.Background()) - if err != nil { - fmt.Printf("failed to start telemetry: %v\n", err) - return nil - } return t } diff --git a/cmd/installer/subcommands/installer/umask_nix.go b/cmd/installer/subcommands/installer/umask_nix.go index 1fd44c01ac405..dec64eccd6860 100644 --- a/cmd/installer/subcommands/installer/umask_nix.go +++ b/cmd/installer/subcommands/installer/umask_nix.go @@ -14,7 +14,7 @@ import ( ) // setInstallerUmask sets umask 0 to override any inherited umask -func setInstallerUmask(span telemetry.Span) { +func setInstallerUmask(span *telemetry.Span) { oldmask := syscall.Umask(0) span.SetTag("inherited_umask", oldmask) } diff --git a/cmd/installer/subcommands/installer/umask_windows.go b/cmd/installer/subcommands/installer/umask_windows.go index 1b076f92bc389..d8661700cd56e 100644 --- a/cmd/installer/subcommands/installer/umask_windows.go +++ b/cmd/installer/subcommands/installer/umask_windows.go @@ -10,4 +10,4 @@ package installer import "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" // setInstallerUmask no-op on Windows -func setInstallerUmask(_ telemetry.Span) {} +func setInstallerUmask(_ *telemetry.Span) {} diff --git a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go index d8631536c3a44..6098dcf98f629 100644 --- a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go +++ b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go @@ -168,11 +168,22 @@ func (wcd *AgentCrashDetect) Run() error { } log.Infof("Sending crash: %v", formatText(crash)) - lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), wcd.tconfig.TelemetryConfig.Endpoints, "ddnpm", true) + lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), toTelemEndpoints(wcd.tconfig.TelemetryConfig.Endpoints), "ddnpm", true) lts.SendLog("WARN", formatText(crash)) return nil } +func toTelemEndpoints(endpoints []*traceconfig.Endpoint) []*internaltelemetry.Endpoint { + telemEndpoints := make([]*internaltelemetry.Endpoint, 0, len(endpoints)) + for _, e := range endpoints { + telemEndpoints = append(telemEndpoints, &internaltelemetry.Endpoint{ + Host: e.Host, + APIKey: e.APIKey, + }) + } + return telemEndpoints +} + func newAgentCrashComponent(deps dependencies) agentcrashdetect.Component { instance := &agentCrashComponent{} instance.tconfig = deps.TConfig.Object() diff --git a/comp/updater/telemetry/telemetryimpl/telemetry.go b/comp/updater/telemetry/telemetryimpl/telemetry.go index d8f961e8398e6..ee1bab4efdd56 100644 --- a/comp/updater/telemetry/telemetryimpl/telemetry.go +++ b/comp/updater/telemetry/telemetryimpl/telemetry.go @@ -7,10 +7,10 @@ package telemetryimpl import ( + "context" "net/http" "go.uber.org/fx" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/updater/telemetry" @@ -38,13 +38,7 @@ func newTelemetry(deps dependencies) (telemetry.Component, error) { client := &http.Client{ Transport: httputils.CreateHTTPTransport(deps.Config), } - telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", - fleettelemetry.WithSamplingRules( - tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1), - tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05), - tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05), - ), - ) - deps.Lc.Append(fx.Hook{OnStart: telemetry.Start, OnStop: telemetry.Stop}) + telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon") + deps.Lc.Append(fx.Hook{OnStop: func(context.Context) error { telemetry.Stop(); return nil }}) return telemetry, nil } diff --git a/pkg/fleet/daemon/daemon.go b/pkg/fleet/daemon/daemon.go index ba79d5456dd3e..5023393d99e5b 100644 --- a/pkg/fleet/daemon/daemon.go +++ b/pkg/fleet/daemon/daemon.go @@ -570,7 +570,7 @@ type requestState struct { ErrorCode installerErrors.InstallerErrorCode } -func newRequestContext(request remoteAPIRequest) (telemetry.Span, context.Context) { +func newRequestContext(request remoteAPIRequest) (*telemetry.Span, context.Context) { ctx := context.WithValue(context.Background(), requestStateKey, &requestState{ Package: request.Package, ID: request.ID, diff --git a/pkg/fleet/installer/setup/common/setup.go b/pkg/fleet/installer/setup/common/setup.go index 0a7b93447c22f..b6e512d7dc5d6 100644 --- a/pkg/fleet/installer/setup/common/setup.go +++ b/pkg/fleet/installer/setup/common/setup.go @@ -41,7 +41,7 @@ type Setup struct { Out *Output Env *env.Env Ctx context.Context - Span telemetry.Span + Span *telemetry.Span Packages Packages Config Config } @@ -130,7 +130,7 @@ func (s *Setup) installPackage(name string, url string) (err error) { span, ctx := telemetry.StartSpanFromContext(s.Ctx, "install") defer func() { span.Finish(err) }() span.SetTag("url", url) - span.SetTag("_top_level", 1) + span.SetTopLevel() s.Out.WriteString(fmt.Sprintf("Installing %s...\n", name)) err = s.installer.Install(ctx, url, nil) diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index 834f54b3d8a53..92f5880723fa9 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -40,7 +40,7 @@ func NewInstallerExec(env *env.Env, installerBinPath string) *InstallerExec { type installerCmd struct { *exec.Cmd - span telemetry.Span + span *telemetry.Span ctx context.Context } diff --git a/pkg/fleet/telemetry/http_wrapper.go b/pkg/fleet/telemetry/http_wrapper.go new file mode 100644 index 0000000000000..2f94bda79fe1f --- /dev/null +++ b/pkg/fleet/telemetry/http_wrapper.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "net/http" + "strconv" + "strings" +) + +// WrapRoundTripper wraps the round tripper with the telemetry round tripper. +func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { + if rt == nil { + rt = http.DefaultTransport + } + if wrapped, ok := rt.(*roundTripper); ok { + rt = wrapped.base + } + return &roundTripper{ + base: rt, + } +} + +type roundTripper struct { + base http.RoundTripper +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + span, _ := StartSpanFromContext(req.Context(), "http.request") + defer func() { span.Finish(err) }() + + url := *req.URL + url.User = nil + + span.span.Type = "http" + span.SetResourceName(req.Method + " " + urlFromRequest(req)) + span.span.Meta["http.method"] = req.Method + span.span.Meta["http.url"] = req.URL.String() + span.span.Meta["span.kind"] = "client" + span.span.Meta["network.destination.name"] = url.Hostname() + res, err = rt.base.RoundTrip(req) + if err != nil { + span.SetTag("http.errors", err.Error()) + return res, err + } + span.SetTag("http.status_code", strconv.Itoa(res.StatusCode)) + if res.StatusCode >= 400 { + span.SetTag("http.errors", res.Status) + } + return res, err +} + +// urlFromRequest returns the URL from the HTTP request. The URL query string is included in the return object iff queryString is true +// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information. +func urlFromRequest(r *http.Request) string { + // Quoting net/http comments about net.Request.URL on server requests: + // "For most requests, fields other than Path and RawQuery will be + // empty. (See RFC 7230, Section 5.3)" + // This is why we don't rely on url.URL.String(), url.URL.Host, url.URL.Scheme, etc... + var url string + path := r.URL.EscapedPath() + scheme := r.URL.Scheme + if r.TLS != nil { + scheme = "https" + } + if r.Host != "" { + url = strings.Join([]string{scheme, "://", r.Host, path}, "") + } else { + url = path + } + // Collect the query string if we are allowed to report it and obfuscate it if possible/allowed + if r.URL.RawQuery != "" { + query := r.URL.RawQuery + url = strings.Join([]string{url, query}, "?") + } + if frag := r.URL.EscapedFragment(); frag != "" { + url = strings.Join([]string{url, frag}, "#") + } + return url +} diff --git a/pkg/fleet/telemetry/span.go b/pkg/fleet/telemetry/span.go index ac8f79516c736..ac5f00b9ade0a 100644 --- a/pkg/fleet/telemetry/span.go +++ b/pkg/fleet/telemetry/span.go @@ -7,20 +7,144 @@ package telemetry import ( - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "context" + "fmt" + "math/rand/v2" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" ) -// Span is an alias for ddtrace.Span until we phase ddtrace out. -type Span struct{ ddtrace.Span } +const spanKey = spanContextKey("span_context") + +type spanContextKey string + +// Span represents a span. +type Span struct { + mu sync.Mutex + span internaltelemetry.Span + finished atomic.Bool +} + +func newSpan(name string, parentID, traceID uint64) *Span { + if traceID == 0 { + traceID = rand.Uint64() + if !headSamplingKeep(name, traceID) { + traceID = dropTraceID + } + } + s := &Span{ + span: internaltelemetry.Span{ + TraceID: traceID, + ParentID: parentID, + SpanID: rand.Uint64(), + Name: name, + Resource: name, + Start: time.Now().UnixNano(), + Meta: make(map[string]string), + Metrics: make(map[string]float64), + }, + } + if parentID == 0 { + s.SetTopLevel() + } + + globalTracer.registerSpan(s) + return s +} // Finish finishes the span with an error. func (s *Span) Finish(err error) { - s.Span.Finish(tracer.WithError(err)) + s.finished.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + s.span.Duration = time.Now().UnixNano() - s.span.Start + if err != nil { + s.span.Error = 1 + s.span.Meta = map[string]string{ + "error.message": err.Error(), + "error.stack": string(debug.Stack()), + } + } + globalTracer.finishSpan(s) } // SetResourceName sets the resource name of the span. func (s *Span) SetResourceName(name string) { - s.Span.SetTag(ext.ResourceName, name) + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.span.Resource = name +} + +// SetTopLevel sets the span as a top level span. +func (s *Span) SetTopLevel() { + s.SetTag("_top_level", 1) +} + +// SetTag sets a tag on the span. +func (s *Span) SetTag(key string, value interface{}) { + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if value == nil { + s.span.Meta[key] = "nil" + } + switch v := value.(type) { + case string: + s.span.Meta[key] = v + case bool: + s.span.Meta[key] = strconv.FormatBool(v) + case int: + s.span.Metrics[key] = float64(v) + case int8: + s.span.Metrics[key] = float64(v) + case int16: + s.span.Metrics[key] = float64(v) + case int32: + s.span.Metrics[key] = float64(v) + case int64: + s.span.Metrics[key] = float64(v) + case uint: + s.span.Metrics[key] = float64(v) + case uint8: + s.span.Metrics[key] = float64(v) + case uint16: + s.span.Metrics[key] = float64(v) + case uint32: + s.span.Metrics[key] = float64(v) + case uint64: + s.span.Metrics[key] = float64(v) + case float32: + s.span.Metrics[key] = float64(v) + case float64: + s.span.Metrics[key] = v + default: + s.span.Meta[key] = fmt.Sprintf("not_supported_type %T", v) + } +} + +type spanIDs struct { + traceID uint64 + spanID uint64 +} + +func getSpanIDsFromContext(ctx context.Context) (spanIDs, bool) { + sIDs, ok := ctx.Value(spanKey).(spanIDs) + if !ok { + return spanIDs{}, false + } + return sIDs, true +} + +func setSpanIDsInContext(ctx context.Context, span *Span) context.Context { + return context.WithValue(ctx, spanKey, spanIDs{traceID: span.span.TraceID, spanID: span.span.SpanID}) } diff --git a/pkg/fleet/telemetry/telemetry.go b/pkg/fleet/telemetry/telemetry.go index 5bbfd8ca773c4..66174ad18fc32 100644 --- a/pkg/fleet/telemetry/telemetry.go +++ b/pkg/fleet/telemetry/telemetry.go @@ -8,280 +8,178 @@ package telemetry import ( "context" - "errors" "fmt" - "io" - "math/rand/v2" - "net" "net/http" "os" "strconv" "strings" - "sync" - - httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - - "github.com/gorilla/mux" + "time" "github.com/DataDog/datadog-agent/pkg/internaltelemetry" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - traceconfig "github.com/DataDog/datadog-agent/pkg/trace/config" - "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" ) const ( - // EnvTraceID is the environment variable key for the trace ID - EnvTraceID = "DATADOG_TRACE_ID" - // EnvParentID is the environment variable key for the parent ID - EnvParentID = "DATADOG_PARENT_ID" -) - -const ( + envTraceID = "DATADOG_TRACE_ID" + envParentID = "DATADOG_PARENT_ID" telemetrySubdomain = "instrumentation-telemetry-intake" - telemetryEndpoint = "/v0.4/traces" ) // Telemetry handles the telemetry for fleet components. type Telemetry struct { telemetryClient internaltelemetry.Client + done chan struct{} + flushed chan struct{} - site string + env string service string - - listener *telemetryListener - server *http.Server - client *http.Client - - samplingRules []tracer.SamplingRule } -// Option is a functional option for telemetry. -type Option func(*Telemetry) - // NewTelemetry creates a new telemetry instance -func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) *Telemetry { - endpoint := &traceconfig.Endpoint{ +func NewTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + t := newTelemetry(client, apiKey, site, service) + t.Start() + return t +} + +func newTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + endpoint := &internaltelemetry.Endpoint{ Host: fmt.Sprintf("https://%s.%s", telemetrySubdomain, strings.TrimSpace(site)), APIKey: apiKey, } - listener := newTelemetryListener() - t := &Telemetry{ - telemetryClient: internaltelemetry.NewClient(client, []*traceconfig.Endpoint{endpoint}, service, site == "datad0g.com"), - site: site, - service: service, - listener: listener, - server: &http.Server{}, - client: &http.Client{ - Transport: &http.Transport{ - Dial: listener.Dial, - }, - }, + env := "prod" + if site == "datad0g.com" { + env = "staging" } - for _, opt := range opts { - opt(t) + + return &Telemetry{ + telemetryClient: internaltelemetry.NewClient(client, []*internaltelemetry.Endpoint{endpoint}, service, site == "datad0g.com"), + done: make(chan struct{}), + flushed: make(chan struct{}), + env: env, + service: service, } - t.server.Handler = t.handler() - return t } // Start starts the telemetry -func (t *Telemetry) Start(_ context.Context) error { +func (t *Telemetry) Start() { + ticker := time.Tick(1 * time.Minute) go func() { - err := t.server.Serve(t.listener) - if err != nil { - log.Infof("telemetry server stopped: %v", err) + for { + select { + case <-ticker: + t.sendCompletedSpans() + case <-t.done: + t.sendCompletedSpans() + close(t.flushed) + return + } } }() - env := "prod" - if t.site == "datad0g.com" { - env = "staging" - } - - tracer.Start( - tracer.WithService(t.service), - tracer.WithServiceVersion(version.AgentVersion), - tracer.WithEnv(env), - tracer.WithGlobalTag("site", t.site), - tracer.WithHTTPClient(t.client), - tracer.WithLogStartup(false), - - // We don't need the value, we just need to enforce that it's not - // the default. If it is, then the tracer will try to use the socket - // if it exists -- and it always exists for newer agents. - // If the agent address is the socket, the tracer overrides WithHTTPClient to use it. - tracer.WithAgentAddr("192.0.2.42:12345"), // 192.0.2.0/24 is reserved - tracer.WithSamplingRules(t.samplingRules), - ) - return nil } // Stop stops the telemetry -func (t *Telemetry) Stop(ctx context.Context) error { - tracer.Flush() - tracer.Stop() - t.listener.Close() - err := t.server.Shutdown(ctx) - if err != nil { - log.Errorf("error shutting down telemetry server: %v", err) - } - return nil -} - -func (t *Telemetry) handler() http.Handler { - r := mux.NewRouter().Headers("Content-Type", "application/msgpack").Subrouter() - r.HandleFunc(telemetryEndpoint, func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - log.Errorf("error reading request body: %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - var traces pb.Traces - _, err = traces.UnmarshalMsg(body) - if err != nil { - log.Errorf("error unmarshalling traces: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - t.telemetryClient.SendTraces(traces) - w.WriteHeader(http.StatusOK) - }) - return r +func (t *Telemetry) Stop() { + close(t.done) + <-t.flushed } -type telemetryListener struct { - conns chan net.Conn - - close chan struct{} - closeOnce sync.Once -} - -func newTelemetryListener() *telemetryListener { - return &telemetryListener{ - conns: make(chan net.Conn), - close: make(chan struct{}), +func (t *Telemetry) extractCompletedSpans() internaltelemetry.Traces { + spans := globalTracer.flushCompletedSpans() + if len(spans) == 0 { + return internaltelemetry.Traces{} } -} - -func (l *telemetryListener) Close() error { - l.closeOnce.Do(func() { - close(l.close) - }) - return nil -} - -func (l *telemetryListener) Accept() (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - case conn := <-l.conns: - return conn, nil + traces := make(map[uint64][]*internaltelemetry.Span) + for _, span := range spans { + span.span.Service = t.service + span.span.Meta["env"] = t.env + span.span.Meta["version"] = version.AgentVersion + span.span.Metrics["_sampling_priority_v1"] = 2 + traces[span.span.TraceID] = append(traces[span.span.TraceID], &span.span) + } + tracesArray := make([]internaltelemetry.Trace, 0, len(traces)) + for _, trace := range traces { + tracesArray = append(tracesArray, internaltelemetry.Trace(trace)) } + return internaltelemetry.Traces(tracesArray) } -func (l *telemetryListener) Addr() net.Addr { - return addr(0) +func (t *Telemetry) sendCompletedSpans() { + tracesArray := t.extractCompletedSpans() + if len(tracesArray) == 0 { + return + } + t.telemetryClient.SendTraces(tracesArray) } -func (l *telemetryListener) Dial(_, _ string) (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - default: +// SpanFromContext returns the span from the context if available. +func SpanFromContext(ctx context.Context) (*Span, bool) { + spanIDs, ok := getSpanIDsFromContext(ctx) + if !ok { + return nil, false } - server, client := net.Pipe() - l.conns <- server - return client, nil + return globalTracer.getSpan(spanIDs.spanID) } -type addr int - -func (addr) Network() string { - return "memory" +// StartSpanFromEnv starts a span using the environment variables to find the parent span. +func StartSpanFromEnv(ctx context.Context, operationName string) (*Span, context.Context) { + traceID, parentID := extractIDsFromEnv() + return StartSpanFromIDs(ctx, operationName, traceID, parentID) } -func (addr) String() string { - return "local" +func extractIDsFromEnv() (string, string) { + parentID, ok := os.LookupEnv(envParentID) + if !ok { + return "0", "0" + } + traceID, ok := os.LookupEnv(envTraceID) + if !ok { + return "0", "0" + } + return traceID, parentID } -// StartSpanFromIDs starts a span using the trace and parent -// IDs provided. -func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - ctxCarrier := tracer.TextMapCarrier{ - tracer.DefaultTraceIDHeader: traceID, - tracer.DefaultParentIDHeader: parentID, - tracer.DefaultPriorityHeader: "2", +func converIDsToUint64(traceID, parentID string) (uint64, uint64) { + traceIDInt, err := strconv.ParseUint(traceID, 10, 64) + if err != nil { + return 0, 0 } - spanCtx, err := tracer.Extract(ctxCarrier) + parentIDInt, err := strconv.ParseUint(parentID, 10, 64) if err != nil { - log.Debugf("failed to extract span context from install script params: %v", err) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return 0, 0 } - spanOptions = append([]ddtrace.StartSpanOption{tracer.ChildOf(spanCtx)}, spanOptions...) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return traceIDInt, parentIDInt } -// SpanFromContext returns the span from the context if available. -func SpanFromContext(ctx context.Context) (Span, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return Span{}, false - } - return Span{span}, true +// StartSpanFromIDs starts a span using the trace and parent +// IDs provided. +func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string) (*Span, context.Context) { + traceIDInt, parentIDInt := converIDsToUint64(traceID, parentID) + span, ctx := startSpanFromIDs(ctx, operationName, traceIDInt, parentIDInt) + span.SetTopLevel() + return span, ctx } -// StartSpanFromContext starts a span using the context to find the parent span. -func StartSpanFromContext(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - span, ctx := tracer.StartSpanFromContext(ctx, operationName, spanOptions...) - return Span{span}, ctx +func startSpanFromIDs(ctx context.Context, operationName string, traceID, parentID uint64) (*Span, context.Context) { + s := newSpan(operationName, parentID, traceID) + ctx = setSpanIDsInContext(ctx, s) + return s, ctx } -// StartSpanFromEnv starts a span using the environment variables to find the parent span. -func StartSpanFromEnv(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - traceID, ok := os.LookupEnv(EnvTraceID) - if !ok { - traceID = strconv.FormatUint(rand.Uint64(), 10) - } - parentID, ok := os.LookupEnv(EnvParentID) - if !ok { - parentID = "0" - } - return StartSpanFromIDs(ctx, operationName, traceID, parentID, spanOptions...) +// StartSpanFromContext starts a span using the context to find the parent span. +func StartSpanFromContext(ctx context.Context, operationName string) (*Span, context.Context) { + spanIDs, _ := getSpanIDsFromContext(ctx) + return startSpanFromIDs(ctx, operationName, spanIDs.traceID, spanIDs.spanID) } // EnvFromContext returns the environment variables for the context. func EnvFromContext(ctx context.Context) []string { - spanCtx, ok := SpanContextFromContext(ctx) + sIDs, ok := getSpanIDsFromContext(ctx) if !ok { return []string{} } return []string{ - fmt.Sprintf("%s=%d", EnvTraceID, spanCtx.TraceID()), - fmt.Sprintf("%s=%d", EnvParentID, spanCtx.SpanID()), + fmt.Sprintf("%s=%s", envTraceID, strconv.FormatUint(sIDs.traceID, 10)), + fmt.Sprintf("%s=%s", envParentID, strconv.FormatUint(sIDs.spanID, 10)), } } - -// SpanContextFromContext extracts the span context from the context if available. -func SpanContextFromContext(ctx context.Context) (ddtrace.SpanContext, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return nil, false - } - return span.Context(), true -} - -// WithSamplingRules sets the sampling rules for the telemetry. -func WithSamplingRules(rules ...tracer.SamplingRule) Option { - return func(t *Telemetry) { - t.samplingRules = rules - } -} - -// WrapRoundTripper wraps the round tripper with the telemetry round tripper. -func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { - return httptrace.WrapRoundTripper(rt) -} diff --git a/pkg/fleet/telemetry/telemetry_test.go b/pkg/fleet/telemetry/telemetry_test.go new file mode 100644 index 0000000000000..d0866ee4d4a13 --- /dev/null +++ b/pkg/fleet/telemetry/telemetry_test.go @@ -0,0 +1,222 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "context" + "errors" + "net/http" + "os" + "testing" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFreshSpan(t *testing.T) { + ctx := context.Background() + _, ok := SpanFromContext(ctx) + require.False(t, ok) + + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + s.SetResourceName("new") + + span, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, s, span) + + assert.Equal(t, "test", s.span.Name) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", span.span.Resource) +} + +func TestInheritence(t *testing.T) { + ctx := context.Background() + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + + child, _ := StartSpanFromContext(ctx, "child") + require.NotNil(t, child) + + assert.Equal(t, s.span.SpanID, child.span.ParentID) + assert.Equal(t, s.span.TraceID, child.span.TraceID) +} + +func TestStartSpanFromIDs(t *testing.T) { + ctx := context.Background() + traceID := "100" + parentID := "200" + + span, ctx := StartSpanFromIDs(ctx, "ids-operation", traceID, parentID) + require.NotNil(t, span, "Expected a span") + require.Equal(t, uint64(100), span.span.TraceID) + require.Equal(t, uint64(200), span.span.ParentID) + + val, ok := span.span.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + + spanFromCtx, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, span, spanFromCtx) +} + +func strPtr(s string) *string { + return &s +} + +func TestSpanFromEnv(t *testing.T) { + randTraceID := uint64(9) + tt := []struct { + name string + envTraceID *string + envParentID *string + expectedTraceID uint64 + expectedParentID uint64 + }{ + { + name: "no parent env", + envTraceID: strPtr("100"), + envParentID: nil, + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "no trace env", + envTraceID: nil, + envParentID: strPtr("100"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "traceID malformed", + envTraceID: strPtr("not-a-number"), + envParentID: strPtr("200"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "parentID malformed", + envTraceID: strPtr("100"), + envParentID: strPtr("not-a-number"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "inheritance", + envTraceID: strPtr("100"), + envParentID: strPtr("200"), + expectedTraceID: 100, + expectedParentID: 200, + }, + } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + if tc.envTraceID != nil { + os.Setenv(envTraceID, *tc.envTraceID) + defer os.Unsetenv(envTraceID) + } + if tc.envParentID != nil { + os.Setenv(envParentID, *tc.envParentID) + defer os.Unsetenv(envParentID) + } + + span, ctx := StartSpanFromEnv(context.Background(), "env-operation") + require.NotNil(t, span, "Expected a span") + s, ok := SpanFromContext(ctx) + assert.True(t, ok) + assert.Equal(t, span, s) + + assert.Equal(t, tc.expectedParentID, span.span.ParentID) + if tc.expectedTraceID != randTraceID { + assert.Equal(t, tc.expectedTraceID, span.span.TraceID) + } else { + assert.NotEqual(t, 0, span.span.TraceID) + } + + }) + } +} + +func TestLimit(t *testing.T) { + totalSpans := maxSpansInFlight + 2 + ctx := context.Background() + for i := 0; i < totalSpans; i++ { + _, ctx = StartSpanFromContext(ctx, "test") + } + assert.Len(t, globalTracer.spans, maxSpansInFlight) +} + +func TestEnvFromContext(t *testing.T) { + s, ctx := StartSpanFromContext(context.Background(), "test") + s.span.TraceID = 456 + s.span.SpanID = 123 + ctx = setSpanIDsInContext(ctx, s) + env := EnvFromContext(ctx) + assert.ElementsMatch(t, []string{"DATADOG_TRACE_ID=456", "DATADOG_PARENT_ID=123"}, env) + + env = EnvFromContext(context.Background()) + assert.ElementsMatch(t, []string{}, env) +} + +func TestSpanFinished(t *testing.T) { + s, _ := StartSpanFromContext(context.Background(), "test") + s.Finish(nil) + s.SetResourceName("new") + s.SetTag("key", "value") + + assert.Equal(t, "test", s.span.Resource) + _, ok := s.span.Meta["key"] + assert.False(t, ok) +} + +func TestRemapOnFlush(t *testing.T) { + const testService = "test-service" + const numTraces = 10 + telem := newTelemetry(&http.Client{}, "api", "datad0g.com", testService) + globalTracer = &tracer{spans: make(map[uint64]*Span)} + + // traces with 2 spans + for i := 0; i < numTraces; i++ { + parentSpan, ctx := StartSpanFromContext(context.Background(), "parent") + childSpan, _ := StartSpanFromContext(ctx, "child") + childSpan.Finish(errors.New("test_error")) + parentSpan.Finish(nil) + } + resTraces := telem.extractCompletedSpans() + require.Len(t, resTraces, numTraces) + + for _, trace := range resTraces { + assert.Len(t, trace, 2) + for _, span := range trace { + assert.Equal(t, testService, span.Service) + assert.Equal(t, "staging", span.Meta["env"]) + assert.Equal(t, 2.0, span.Metrics["_sampling_priority_v1"]) + } + var parent, child *internaltelemetry.Span + if trace[0].Name == "parent" { + parent = trace[0] + child = trace[1] + } else { + parent = trace[1] + child = trace[0] + } + assert.Equal(t, parent.SpanID, child.ParentID) + val, ok := parent.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + _, ok = child.Metrics["_top_level"] + require.False(t, ok) + + require.Equal(t, int32(1), child.Error) + require.Equal(t, "test_error", child.Meta["error.message"]) + require.Contains(t, child.Meta["error.stack"], "telemetry_test.go") + } +} diff --git a/pkg/fleet/telemetry/tracer.go b/pkg/fleet/telemetry/tracer.go new file mode 100644 index 0000000000000..0923b3563e457 --- /dev/null +++ b/pkg/fleet/telemetry/tracer.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "math" + "strings" + "sync" +) + +const ( + dropTraceID = 1 + maxSpansInFlight = 1000 +) + +var ( + globalTracer *tracer + samplingRates = map[string]float64{ + "cdn": 0.1, + "garbage_collect": 0.05, + "HTTPClient": 0.05, + } +) + +func init() { + globalTracer = &tracer{ + spans: make(map[uint64]*Span), + } +} + +type tracer struct { + mu sync.Mutex + spans map[uint64]*Span + completedSpans []*Span +} + +func (t *tracer) registerSpan(span *Span) { + if span.span.TraceID == dropTraceID { + return + } + t.mu.Lock() + defer t.mu.Unlock() + // naive maxSpansInFlight check as this is just telemetry + // next iteration if needed would be to flush long running spans to troubleshoot + if len(t.spans) >= maxSpansInFlight { + return + } + t.spans[span.span.SpanID] = span +} + +func (t *tracer) getSpan(spanID uint64) (*Span, bool) { + t.mu.Lock() + defer t.mu.Unlock() + span, ok := t.spans[spanID] + return span, ok +} + +func (t *tracer) finishSpan(span *Span) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.spans, span.span.SpanID) + t.completedSpans = append(t.completedSpans, span) +} + +func (t *tracer) flushCompletedSpans() []*Span { + t.mu.Lock() + defer t.mu.Unlock() + newSpanArray := make([]*Span, 0) + completedSpans := t.completedSpans + t.completedSpans = newSpanArray + return completedSpans +} + +func headSamplingKeep(spanName string, traceID uint64) bool { + for k, r := range samplingRates { + if strings.Contains(spanName, k) { + return sampledByRate(traceID, r) + } + } + return true +} + +func sampledByRate(n uint64, rate float64) bool { + if rate < 1 { + return n*uint64(1111111111111111111) < uint64(rate*math.MaxUint64) + } + return true +} diff --git a/pkg/internaltelemetry/client.go b/pkg/internaltelemetry/client.go index 52b1b2010edf0..ee7754fad190b 100644 --- a/pkg/internaltelemetry/client.go +++ b/pkg/internaltelemetry/client.go @@ -21,8 +21,6 @@ import ( "go.uber.org/atomic" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - "github.com/DataDog/datadog-agent/pkg/trace/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" "github.com/shirou/gopsutil/v4/host" @@ -36,13 +34,19 @@ const ( // Client defines the interface for a telemetry client type Client interface { SendLog(level string, message string) - SendTraces(traces pb.Traces) + SendTraces(traces Traces) +} + +// Endpoint defines the endpoint object +type Endpoint struct { + APIKey string `json:"-"` + Host string } type client struct { m sync.Mutex client httpClient - endpoints []*config.Endpoint + endpoints []*Endpoint sendPayloadTimeout time.Duration // we can pre-calculate the host payload structure at init time @@ -94,7 +98,7 @@ type Application struct { // TracePayload defines the trace payload object type TracePayload struct { - Traces []pb.Trace `json:"traces"` + Traces []Trace `json:"traces"` } // LogPayload defines the log payload object @@ -115,7 +119,7 @@ type httpClient interface { } // NewClient creates a new telemetry client -func NewClient(httpClient httpClient, endpoints []*config.Endpoint, service string, debug bool) Client { +func NewClient(httpClient httpClient, endpoints []*Endpoint, service string, debug bool) Client { info, err := host.Info() if err != nil { log.Errorf("failed to retrieve host info: %v", err) @@ -158,7 +162,7 @@ func (c *client) SendLog(level, message string) { c.sendPayload(RequestTypeLogs, payload) } -func (c *client) SendTraces(traces pb.Traces) { +func (c *client) SendTraces(traces Traces) { c.m.Lock() defer c.m.Unlock() payload := TracePayload{ @@ -170,8 +174,8 @@ func (c *client) SendTraces(traces pb.Traces) { // sampleTraces is a simple uniform sampling function that samples traces based // on the sampling rate, given that there is no trace agent to sample the traces // We try to keep the tracer behaviour: the first rule that matches apply its rate to the whole trace -func (c *client) sampleTraces(traces pb.Traces) pb.Traces { - tracesWithSampling := pb.Traces{} +func (c *client) sampleTraces(traces Traces) Traces { + tracesWithSampling := Traces{} for _, trace := range traces { samplingRate := 1.0 for _, span := range trace { @@ -206,7 +210,7 @@ func (c *client) sendPayload(requestType RequestType, payload interface{}) { group := sync.WaitGroup{} for _, endpoint := range c.endpoints { group.Add(1) - go func(endpoint *config.Endpoint) { + go func(endpoint *Endpoint) { defer group.Done() url := fmt.Sprintf("%s%s", endpoint.Host, telemetryEndpoint) req, err := http.NewRequest("POST", url, bytes.NewReader(serializedPayload)) diff --git a/pkg/internaltelemetry/traces.go b/pkg/internaltelemetry/traces.go new file mode 100644 index 0000000000000..54c8545a604b7 --- /dev/null +++ b/pkg/internaltelemetry/traces.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +// Package internaltelemetry full description in README.md +package internaltelemetry + +// Traces is a collection of traces +type Traces []Trace + +// Trace is a collection of spans with the same trace ID +type Trace []*Span + +// Span used for installation telemetry +type Span struct { + // Service is the name of the service that handled this span. + Service string `json:"service"` + // Name is the name of the operation this span represents. + Name string `json:"name"` + // Resource is the name of the resource this span represents. + Resource string `json:"resource"` + // TraceID is the ID of the trace to which this span belongs. + TraceID uint64 `json:"trace_id"` + // SpanID is the ID of this span. + SpanID uint64 `json:"span_id"` + // ParentID is the ID of the parent span. + ParentID uint64 `json:"parent_id"` + // Start is the start time of this span in nanoseconds since the Unix epoch. + Start int64 `json:"start"` + // Duration is the duration of this span in nanoseconds. + Duration int64 `json:"duration"` + // Error is the error status of this span. + Error int32 `json:"error"` + // Meta is a mapping from tag name to tag value for string-valued tags. + Meta map[string]string `json:"meta,omitempty"` + // Metrics is a mapping from metric name to metric value for numeric metrics. + Metrics map[string]float64 `json:"metrics,omitempty"` + // Type is the type of the span. + Type string `json:"type"` +} From f236edf97bffd669b155ef3bef109563a5690afe Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 18 Dec 2024 19:33:36 +0100 Subject: [PATCH 13/26] upgrade(installer): Retry more network errors (#32346) --- pkg/fleet/installer/oci/download.go | 127 +++++++++++++++++----------- 1 file changed, 79 insertions(+), 48 deletions(-) diff --git a/pkg/fleet/installer/oci/download.go b/pkg/fleet/installer/oci/download.go index 7fcce0234512c..e557cf8b53d7f 100644 --- a/pkg/fleet/installer/oci/download.go +++ b/pkg/fleet/installer/oci/download.go @@ -62,8 +62,8 @@ const ( ) const ( - layerMaxSize = 3 << 30 // 3GiB - extractLayerRetries = 3 + layerMaxSize = 3 << 30 // 3GiB + networkRetries = 3 ) var ( @@ -318,30 +318,32 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) return fmt.Errorf("could not get layer media type: %w", err) } if layerMediaType == mediaType { - // Retry stream reset errors - for i := 0; i < extractLayerRetries; i++ { - if i > 0 { - time.Sleep(time.Second) - } - uncompressedLayer, err := layer.Uncompressed() - if err != nil { - return fmt.Errorf("could not uncompress layer: %w", err) - } - err = tar.Extract(uncompressedLayer, dir, layerMaxSize) - uncompressedLayer.Close() - if err != nil { - if !isStreamResetError(err) && !isConnectionResetByPeerError(err) { - return fmt.Errorf("could not extract layer: %w", err) + err = withNetworkRetries( + func() error { + var err error + defer func() { + if err != nil { + deferErr := tar.Clean(dir) + if deferErr != nil { + err = deferErr + } + } + }() + uncompressedLayer, err := layer.Uncompressed() + if err != nil { + return err } - log.Warnf("network error while extracting layer, retrying") - // Clean up the directory before retrying to avoid partial extraction - err = tar.Clean(dir) + err = tar.Extract(uncompressedLayer, dir, layerMaxSize) + uncompressedLayer.Close() if err != nil { - return fmt.Errorf("could not clean directory: %w", err) + return err } - } else { - break - } + + return nil + }, + ) + if err != nil { + return fmt.Errorf("could not extract layer: %w", err) } } } @@ -349,16 +351,22 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) } // WriteOCILayout writes the image as an OCI layout to the given directory. -func (d *DownloadedPackage) WriteOCILayout(dir string) error { - layoutPath, err := layout.Write(dir, empty.Index) - if err != nil { - return fmt.Errorf("could not write layout: %w", err) - } - err = layoutPath.AppendImage(d.Image) - if err != nil { - return fmt.Errorf("could not append image to layout: %w", err) - } - return nil +func (d *DownloadedPackage) WriteOCILayout(dir string) (err error) { + var layoutPath layout.Path + return withNetworkRetries( + func() error { + layoutPath, err = layout.Write(dir, empty.Index) + if err != nil { + return fmt.Errorf("could not write layout: %w", err) + } + + err = layoutPath.AppendImage(d.Image) + if err != nil { + return fmt.Errorf("could not append image to layout: %w", err) + } + return nil + }, + ) } // PackageURL returns the package URL for the given site, package and version. @@ -371,15 +379,50 @@ func PackageURL(env *env.Env, pkg string, version string) string { } } +func withNetworkRetries(f func() error) error { + var err error + for i := 0; i < networkRetries; i++ { + err = f() + if err == nil { + return nil + } + if !isRetryableNetworkError(err) { + return err + } + log.Warnf("retrying after network error: %s", err) + time.Sleep(time.Second) + } + return err +} + +// isRetryableNetworkError returns true if the error is a network error we should retry on +func isRetryableNetworkError(err error) bool { + if err == nil { + return false + } + + if netErr, ok := err.(*net.OpError); ok { + if netErr.Temporary() { + // Temporary errors, such as "connection timed out" + return true + } + if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { + if errno, ok := syscallErr.Err.(syscall.Errno); ok { + // Connection reset errors, such as "connection reset by peer" + return errno == syscall.ECONNRESET + } + } + } + + return isStreamResetError(err) +} + // isStreamResetError returns true if the given error is a stream reset error. // Sometimes, in GCR, the tar extract fails with "stream error: stream ID x; INTERNAL_ERROR; received from peer". // This happens because the uncompressed layer reader is a http/2 response body under the hood. That body is // streamed and receives a "reset stream frame", with the code 0x2 (INTERNAL_ERROR). This is an error from the server // that we need to retry. func isStreamResetError(err error) bool { - if err == nil { - return false - } serr := http2.StreamError{} if errors.As(err, &serr) { return serr.Code == http2.ErrCodeInternal @@ -391,18 +434,6 @@ func isStreamResetError(err error) bool { return false } -// isConnectionResetByPeer returns true if the error is a connection reset by peer error -func isConnectionResetByPeerError(err error) bool { - if netErr, ok := err.(*net.OpError); ok { - if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { - if errno, ok := syscallErr.Err.(syscall.Errno); ok { - return errno == syscall.ECONNRESET - } - } - } - return false -} - type usernamePasswordKeychain struct { username string password string From 86c077224161ff41e04f438e7ddb0cc9152af0a1 Mon Sep 17 00:00:00 2001 From: Zhengda Lu Date: Wed, 18 Dec 2024 13:33:49 -0500 Subject: [PATCH 14/26] bump github.com/DataDog/go-sqllexer to v0.0.18 (#32315) --- comp/otelcol/ddflareextension/impl/go.mod | 2 +- comp/otelcol/ddflareextension/impl/go.sum | 4 ++-- comp/otelcol/otlp/components/exporter/datadogexporter/go.mod | 2 +- comp/otelcol/otlp/components/exporter/datadogexporter/go.sum | 4 ++-- comp/otelcol/otlp/components/statsprocessor/go.mod | 2 +- comp/otelcol/otlp/components/statsprocessor/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- pkg/config/remote/go.mod | 2 +- pkg/config/remote/go.sum | 4 ++-- pkg/obfuscate/go.mod | 2 +- pkg/obfuscate/go.sum | 4 ++-- pkg/trace/go.mod | 2 +- pkg/trace/go.sum | 4 ++-- pkg/trace/stats/oteltest/go.mod | 2 +- pkg/trace/stats/oteltest/go.sum | 4 ++-- test/otel/go.mod | 2 +- test/otel/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 21d3f17b9066a..ce08a7999f072 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -267,7 +267,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index d4f8e67e79da5..7aed291a64674 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -70,8 +70,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index 301ef0f3b1f48..dfe091b9d4b79 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -201,7 +201,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index e9c507fbafa8a..c32ab7009cbeb 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 18606500a04d2..c03fc39e3ca59 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -43,7 +43,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index a19fb370fa6c4..22ef689018918 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/go.mod b/go.mod index c8de7db9cc5f0..959651ab0bcce 100644 --- a/go.mod +++ b/go.mod @@ -735,7 +735,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/DataDog/go-libddwaf/v3 v3.5.1 - github.com/DataDog/go-sqllexer v0.0.17 + github.com/DataDog/go-sqllexer v0.0.18 github.com/Datadog/dublin-traceroute v0.0.2 github.com/aquasecurity/trivy v0.49.2-0.20240227072422-e1ea02c7b80d github.com/aws/aws-sdk-go-v2/service/kms v1.37.6 diff --git a/go.sum b/go.sum index 26375333ef0cf..ffa368593f411 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302 github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302-b9cf785c02fe/go.mod h1:90sqV0j7E8wYCyqIp5d9HmYWLTFQttqPFFtNYDyAybQ= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index a1c4a4ebbc561..72ed5685445f8 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -87,7 +87,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index ca537b0d1660e..512cb6bacbab2 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -11,8 +11,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index b34df591521b5..709579d414b7f 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/DataDog/datadog-go/v5 v5.6.0 - github.com/DataDog/go-sqllexer v0.0.17 + github.com/DataDog/go-sqllexer v0.0.18 github.com/outcaste-io/ristretto v0.2.3 github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index 063bd88a07005..940c4e5438e4a 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 29bf241afd7b8..c3f102a944494 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -61,7 +61,7 @@ require go.opentelemetry.io/collector/processor v0.115.0 // indirect require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/zstd v1.5.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index e1b7335487809..efd28e50f6d16 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 071e7c2c2eba0..1259dbf55e8bf 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -30,7 +30,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index a19fb370fa6c4..22ef689018918 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/test/otel/go.mod b/test/otel/go.mod index 88d5f3a127be5..beb5f560e099e 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -179,7 +179,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index bce88ff15bfa8..4ccec6f105aa4 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.22.0 h1:r1Dx2cRHCBWkVluSZA41i4eoI/nOGbcrrZdkqWjoFCc= From 73f4ec93904fed8ed102a15fba2132199a012c04 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 18 Dec 2024 10:47:21 -0800 Subject: [PATCH 15/26] Update lading to 0.25.2 (#32271) Signed-off-by: Brian L. Troutwine --- test/regression/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/regression/config.yaml b/test/regression/config.yaml index 362301357f864..4dcef653249aa 100644 --- a/test/regression/config.yaml +++ b/test/regression/config.yaml @@ -1,5 +1,5 @@ lading: - version: 0.24.0 + version: 0.25.2 target: cpu_allotment: 8 From 32aabeeeca86051037569dfe8a0ab411124ab09d Mon Sep 17 00:00:00 2001 From: eugene kirillov <3404064+krlv@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:05:57 -0800 Subject: [PATCH 16/26] Bump otel-agent BYOC flow to use 7.59.0-v1.1.0 agent version (#31986) --- Dockerfiles/agent-ot/Dockerfile.agent-otel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/agent-ot/Dockerfile.agent-otel b/Dockerfiles/agent-ot/Dockerfile.agent-otel index 78534598a581c..49ea309af7019 100644 --- a/Dockerfiles/agent-ot/Dockerfile.agent-otel +++ b/Dockerfiles/agent-ot/Dockerfile.agent-otel @@ -1,5 +1,5 @@ -ARG AGENT_VERSION=7.57.0-v1.0-ot-beta-jmx -ARG AGENT_BRANCH=7.57.x-otel-beta-v1 +ARG AGENT_VERSION=7.59.0-v1.1.0-ot-beta-jmx +ARG AGENT_BRANCH=7.59.x # Use the Ubuntu Slim AMD64 base image FROM ubuntu:24.04 AS builder From 47cad73468d7a08c8319e24440326659dfe6992b Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 18 Dec 2024 19:06:08 +0000 Subject: [PATCH 17/26] [APR-190] Change the default metric compression kind to be `zstd`. (#32087) Co-authored-by: blt Co-authored-by: scottopell --- pkg/config/setup/config.go | 2 +- .../internal/metrics/service_checks_test.go | 2 +- ...s-compression-default-zstd-c786c2d28eb51b1f.yaml | 13 +++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 24f8d81503459..fdb1d427e9d15 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -79,7 +79,7 @@ const ( DefaultRuntimePoliciesDir = "/etc/datadog-agent/runtime-security.d" // DefaultCompressorKind is the default compressor. Options available are 'zlib' and 'zstd' - DefaultCompressorKind = "zlib" + DefaultCompressorKind = "zstd" // DefaultZstdCompressionLevel is the default compression level for `zstd`. // Compression level 1 provides the lowest compression ratio, but uses much less RSS especially diff --git a/pkg/serializer/internal/metrics/service_checks_test.go b/pkg/serializer/internal/metrics/service_checks_test.go index 2a96f3d10072a..2559d7c9ed2cc 100644 --- a/pkg/serializer/internal/metrics/service_checks_test.go +++ b/pkg/serializer/internal/metrics/service_checks_test.go @@ -126,7 +126,7 @@ func TestPayloadsEmptyServiceCheck(t *testing.T) { func TestPayloadsServiceChecks(t *testing.T) { config := mock.New(t) - config.Set("serializer_max_payload_size", 200, pkgconfigmodel.SourceAgentRuntime) + config.Set("serializer_max_payload_size", 250, pkgconfigmodel.SourceAgentRuntime) serviceCheckCollection := []ServiceChecks{ {createServiceCheck("1"), createServiceCheck("2"), createServiceCheck("3")}, diff --git a/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml new file mode 100644 index 0000000000000..7d9a8653d8f0d --- /dev/null +++ b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Metric payloads are compressed using `zstd` compression by default. + This can be reverted to the previous compression kind by adding + ``serializer_compressor_kind: zlib`` to the configuration. From ba0629fd7798a2e063213a5af520a621d9e0d163 Mon Sep 17 00:00:00 2001 From: Raphael Gavache Date: Wed, 18 Dec 2024 20:06:15 +0100 Subject: [PATCH 18/26] [fleet] fix codeowner (#32357) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ec5ac92f6a2e5..50f8ff18a0c96 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -176,6 +176,7 @@ /.gitlab/package_build/ @DataDog/agent-delivery /.gitlab/package_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/package_build/installer.yml @DataDog/agent-delivery @DataDog/fleet /.gitlab/packaging/ @DataDog/agent-delivery /.gitlab/benchmarks/benchmarks.yml @DataDog/agent-apm From 7c1dae666e77bddee4ded3e2cbb5f822bb39764b Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos <91925154+gabedos@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:21:36 -0500 Subject: [PATCH 19/26] [CONTP-517] Support None card in tagger (#31897) --- .../tagger/collectors/pod_tag_extractor.go | 2 + comp/core/tagger/impl/tagger.go | 5 +-- .../subscriber/subscription_manager_test.go | 37 +++++++++++++++++++ comp/core/tagger/tagstore/entity_tags.go | 12 ++++-- comp/core/tagger/tagstore/entity_tags_test.go | 6 +++ comp/core/tagger/tagstore/tagstore_test.go | 4 ++ comp/core/tagger/telemetry/telemetry.go | 4 ++ comp/core/tagger/types/types.go | 16 ++++++++ ...none-cardinality-tag-f2ceec7ec571387e.yaml | 11 ++++++ 9 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml diff --git a/comp/core/tagger/collectors/pod_tag_extractor.go b/comp/core/tagger/collectors/pod_tag_extractor.go index 669a71584f609..aded502edaf4c 100644 --- a/comp/core/tagger/collectors/pod_tag_extractor.go +++ b/comp/core/tagger/collectors/pod_tag_extractor.go @@ -34,6 +34,8 @@ func (p *PodTagExtractor) Extract(podEntity *workloadmeta.KubernetesPod, cardina return append(tagInfos.LowCardTags, tagInfos.OrchestratorCardTags...) case types.LowCardinality: return tagInfos.LowCardTags + case types.NoneCardinality: + return []string{} default: log.Errorf("unsupported tag cardinality %v", cardinality) return []string{} diff --git a/comp/core/tagger/impl/tagger.go b/comp/core/tagger/impl/tagger.go index 7023106b1c118..de05b9f6eaa09 100644 --- a/comp/core/tagger/impl/tagger.go +++ b/comp/core/tagger/impl/tagger.go @@ -422,7 +422,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert // | none | empty || empty | // | empty | not empty || container prefix + originFromMsg | // | none | not empty || container prefix + originFromMsg | - if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == "none" { + if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" @@ -460,8 +460,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert } default: // Disable origin detection if cardinality is none - // TODO: The `none` cardinality should be directly supported by the Tagger. - if originInfo.Cardinality == "none" { + if originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" diff --git a/comp/core/tagger/subscriber/subscription_manager_test.go b/comp/core/tagger/subscriber/subscription_manager_test.go index b9684af30e2d4..dbebd8ed23d98 100644 --- a/comp/core/tagger/subscriber/subscription_manager_test.go +++ b/comp/core/tagger/subscriber/subscription_manager_test.go @@ -109,6 +109,21 @@ func TestSubscriptionManager(t *testing.T) { highCardSubscription.Unsubscribe() + // None Cardinality Subscriber + noneCardSubID := "none-card-sub" + noneCardSubscription, err := sm.Subscribe(noneCardSubID, types.NewFilterBuilder().Include(types.EntityIDPrefix("foo")).Build(types.NoneCardinality), nil) + require.NoError(t, err) + + sm.Notify([]types.EntityEvent{ + events["added"], + events["modified"], + events["deleted"], + events["added-with-no-id"], + events["added-with-unmatched-prefix"], + }) + + noneCardSubscription.Unsubscribe() + // Verify low cardinality subscriber received events assertReceivedEvents(t, lowCardSubscription.EventsChan(), []types.EntityEvent{ { @@ -192,6 +207,28 @@ func TestSubscriptionManager(t *testing.T) { }, }, }) + + // Verify none cardinality subscriber received events + assertReceivedEvents(t, noneCardSubscription.EventsChan(), []types.EntityEvent{ + { + EventType: types.EventTypeAdded, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeModified, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeDeleted, + Entity: types.Entity{ + ID: entityID, + }, + }, + }) } func assertReceivedEvents(t *testing.T, ch chan []types.EntityEvent, expectedEvents []types.EntityEvent) { diff --git a/comp/core/tagger/tagstore/entity_tags.go b/comp/core/tagger/tagstore/entity_tags.go index 3488a8e345c16..ed62d45849109 100644 --- a/comp/core/tagger/tagstore/entity_tags.go +++ b/comp/core/tagger/tagstore/entity_tags.go @@ -109,12 +109,16 @@ func (e *EntityTagsWithMultipleSources) getStandard() []string { func (e *EntityTagsWithMultipleSources) getHashedTags(cardinality types.TagCardinality) tagset.HashedTags { e.computeCache() - if cardinality == types.HighCardinality { + switch cardinality { + case types.HighCardinality: return e.cachedAll - } else if cardinality == types.OrchestratorCardinality { + case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} + default: + return e.cachedLow } - return e.cachedLow } func (e *EntityTagsWithMultipleSources) computeCache() { @@ -302,6 +306,8 @@ func (e *EntityTagsWithSingleSource) getHashedTags(cardinality types.TagCardinal return e.cachedAll case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} default: return e.cachedLow } diff --git a/comp/core/tagger/tagstore/entity_tags_test.go b/comp/core/tagger/tagstore/entity_tags_test.go index c93d764377aaf..16e2d4b077f86 100644 --- a/comp/core/tagger/tagstore/entity_tags_test.go +++ b/comp/core/tagger/tagstore/entity_tags_test.go @@ -100,6 +100,12 @@ func TestGetHashedTags(t *testing.T) { []string{"l1:v1", "l2:v2", "service:s1", "o1:v1", "o2:v2", "h1:v1", "h2:v2"}, entityTags.getHashedTags(types.HighCardinality).Get(), ) + + assert.Equal( + t, + []string(nil), + entityTags.getHashedTags(types.NoneCardinality).Get(), + ) } func TestTagsForSource(t *testing.T) { diff --git a/comp/core/tagger/tagstore/tagstore_test.go b/comp/core/tagger/tagstore/tagstore_test.go index 4699aa6ad56cb..0d992b682ca68 100644 --- a/comp/core/tagger/tagstore/tagstore_test.go +++ b/comp/core/tagger/tagstore/tagstore_test.go @@ -88,10 +88,12 @@ func (s *StoreTestSuite) TestLookup() { tagsHigh := s.tagstore.Lookup(entityID, types.HighCardinality) tagsOrch := s.tagstore.Lookup(entityID, types.OrchestratorCardinality) tagsLow := s.tagstore.Lookup(entityID, types.LowCardinality) + tagsNone := s.tagstore.Lookup(entityID, types.NoneCardinality) assert.Len(s.T(), tagsHigh, 4) assert.Len(s.T(), tagsLow, 2) assert.Len(s.T(), tagsOrch, 3) + assert.Nil(s.T(), tagsNone) } func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { @@ -118,10 +120,12 @@ func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { tagsLow := s.tagstore.LookupHashedWithEntityStr(entityID, types.LowCardinality) tagsOrch := s.tagstore.LookupHashedWithEntityStr(entityID, types.OrchestratorCardinality) tagsHigh := s.tagstore.LookupHashedWithEntityStr(entityID, types.HighCardinality) + tagsNone := s.tagstore.LookupHashedWithEntityStr(entityID, types.NoneCardinality) assert.ElementsMatch(s.T(), tagsLow.Get(), []string{"low1", "low2"}) assert.ElementsMatch(s.T(), tagsOrch.Get(), []string{"low1", "low2", "orchestrator1"}) assert.ElementsMatch(s.T(), tagsHigh.Get(), []string{"low1", "low2", "orchestrator1", "high1"}) + assert.ElementsMatch(s.T(), tagsNone.Get(), []string{}) } func (s *StoreTestSuite) TestLookupStandard() { diff --git a/comp/core/tagger/telemetry/telemetry.go b/comp/core/tagger/telemetry/telemetry.go index 97c7153d9c052..0682475eb6d8e 100644 --- a/comp/core/tagger/telemetry/telemetry.go +++ b/comp/core/tagger/telemetry/telemetry.go @@ -68,6 +68,7 @@ type Store struct { LowCardinalityQueries CardinalityTelemetry OrchestratorCardinalityQueries CardinalityTelemetry HighCardinalityQueries CardinalityTelemetry + NoneCardinalityQueries CardinalityTelemetry UnknownCardinalityQueries CardinalityTelemetry } @@ -144,6 +145,7 @@ func NewStore(telemetryComp telemetry.Component) *Store { LowCardinalityQueries: newCardinalityTelemetry(queries, types.LowCardinalityString), OrchestratorCardinalityQueries: newCardinalityTelemetry(queries, types.OrchestratorCardinalityString), HighCardinalityQueries: newCardinalityTelemetry(queries, types.HighCardinalityString), + NoneCardinalityQueries: newCardinalityTelemetry(queries, types.NoneCardinalityString), UnknownCardinalityQueries: newCardinalityTelemetry(queries, types.UnknownCardinalityString), } }) @@ -160,6 +162,8 @@ func (s *Store) QueriesByCardinality(card types.TagCardinality) *CardinalityTele return &s.OrchestratorCardinalityQueries case types.HighCardinality: return &s.HighCardinalityQueries + case types.NoneCardinality: + return &s.NoneCardinalityQueries default: return &s.UnknownCardinalityQueries } diff --git a/comp/core/tagger/types/types.go b/comp/core/tagger/types/types.go index e45f9e6daa084..dc53af7baf27b 100644 --- a/comp/core/tagger/types/types.go +++ b/comp/core/tagger/types/types.go @@ -78,6 +78,7 @@ const ( LowCardinality TagCardinality = iota OrchestratorCardinality HighCardinality + NoneCardinality ) // Entity is an entity ID + tags. @@ -92,6 +93,10 @@ type Entity struct { // GetTags flattens all tags from all cardinalities into a single slice of tag // strings. func (e Entity) GetTags(cardinality TagCardinality) []string { + if cardinality == NoneCardinality { + return []string{} + } + tagArrays := make([][]string, 0, 3) tagArrays = append(tagArrays, e.LowCardinalityTags) @@ -117,6 +122,11 @@ func (e Entity) Copy(cardinality TagCardinality) Entity { case LowCardinality: newEntity.HighCardinalityTags = nil newEntity.OrchestratorCardinalityTags = nil + case NoneCardinality: + newEntity.HighCardinalityTags = nil + newEntity.OrchestratorCardinalityTags = nil + newEntity.LowCardinalityTags = nil + newEntity.StandardTags = nil } return newEntity @@ -131,6 +141,8 @@ const ( ShortOrchestratorCardinalityString = "orch" // HighCardinalityString is the string representation of the high cardinality HighCardinalityString = "high" + // NoneCardinalityString is the string representation of the none cardinality + NoneCardinalityString = "none" // UnknownCardinalityString represents an unknown level of cardinality UnknownCardinalityString = "unknown" ) @@ -145,6 +157,8 @@ func StringToTagCardinality(c string) (TagCardinality, error) { return OrchestratorCardinality, nil case LowCardinalityString: return LowCardinality, nil + case NoneCardinalityString: + return NoneCardinality, nil default: return LowCardinality, fmt.Errorf("unsupported value %s received for tag cardinality", c) } @@ -160,6 +174,8 @@ func TagCardinalityToString(c TagCardinality) string { return OrchestratorCardinalityString case LowCardinality: return LowCardinalityString + case NoneCardinality: + return NoneCardinalityString default: return UnknownCardinalityString } diff --git a/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml new file mode 100644 index 0000000000000..2957ae7f9441d --- /dev/null +++ b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Adds support for the `none` cardinality type in the tagger. From a2eda15d0208e92ed2288b0efcd05808bb23236c Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Wed, 18 Dec 2024 20:35:49 +0100 Subject: [PATCH 20/26] [CWS] cut allocation in `GetProcContainerContext` (#32183) --- pkg/security/utils/cgroup.go | 53 ++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/pkg/security/utils/cgroup.go b/pkg/security/utils/cgroup.go index 5dbbb97c58af1..df59e317006a5 100644 --- a/pkg/security/utils/cgroup.go +++ b/pkg/security/utils/cgroup.go @@ -12,6 +12,7 @@ import ( "bufio" "bytes" "crypto/sha256" + "fmt" "os" "strconv" "strings" @@ -49,6 +50,49 @@ func (cg ControlGroup) GetContainerID() containerutils.ContainerID { return containerutils.ContainerID(id) } +// GetLastProcControlGroups returns the first cgroup membership of the specified task. +func GetLastProcControlGroups(tgid, pid uint32) (ControlGroup, error) { + data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) + if err != nil { + return ControlGroup{}, err + } + + data = bytes.TrimSpace(data) + + index := bytes.LastIndexByte(data, '\n') + if index < 0 { + index = 0 + } else { + index++ // to skip the \n + } + if index >= len(data) { + return ControlGroup{}, fmt.Errorf("invalid cgroup data: %s", data) + } + + lastLine := string(data[index:]) + + idstr, rest, ok := strings.Cut(lastLine, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + id, err := strconv.Atoi(idstr) + if err != nil { + return ControlGroup{}, err + } + + controllers, path, ok := strings.Cut(rest, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + return ControlGroup{ + ID: id, + Controllers: strings.Split(controllers, ","), + Path: path, + }, nil +} + // GetProcControlGroups returns the cgroup membership of the specified task. func GetProcControlGroups(tgid, pid uint32) ([]ControlGroup, error) { data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) @@ -85,15 +129,14 @@ func GetProcContainerID(tgid, pid uint32) (containerutils.ContainerID, error) { // GetProcContainerContext returns the container ID which the process belongs to along with its manager. Returns "" if the process does not belong // to a container. func GetProcContainerContext(tgid, pid uint32) (containerutils.ContainerID, model.CGroupContext, error) { - cgroups, err := GetProcControlGroups(tgid, pid) - if err != nil || len(cgroups) == 0 { + cgroup, err := GetLastProcControlGroups(tgid, pid) + if err != nil { return "", model.CGroupContext{}, err } - lastCgroup := len(cgroups) - 1 - containerID, runtime := cgroups[lastCgroup].GetContainerContext() + containerID, runtime := cgroup.GetContainerContext() cgroupContext := model.CGroupContext{ - CGroupID: containerutils.CGroupID(cgroups[lastCgroup].Path), + CGroupID: containerutils.CGroupID(cgroup.Path), CGroupFlags: runtime, } From 23a03806e9998192fbe5fcd1d38e193acf8d7713 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 18 Dec 2024 14:49:23 -0500 Subject: [PATCH 21/26] [SLES-2001] decode lambda error.msg and error.type (#32231) --- pkg/serverless/daemon/routes.go | 10 ++++++ pkg/serverless/daemon/routes_test.go | 48 +++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/pkg/serverless/daemon/routes.go b/pkg/serverless/daemon/routes.go index 733b4a49b050c..fe78651ac4d2f 100644 --- a/pkg/serverless/daemon/routes.go +++ b/pkg/serverless/daemon/routes.go @@ -104,7 +104,17 @@ func (e *EndInvocation) ServeHTTP(w http.ResponseWriter, r *http.Request) { } errorMsg := r.Header.Get(invocationlifecycle.InvocationErrorMsgHeader) + if decodedMsg, err := base64.StdEncoding.DecodeString(errorMsg); err != nil { + log.Debug("Error message header may not be encoded, setting as is") + } else { + errorMsg = string(decodedMsg) + } errorType := r.Header.Get(invocationlifecycle.InvocationErrorTypeHeader) + if decodedType, err := base64.StdEncoding.DecodeString(errorType); err != nil { + log.Debug("Error type header may not be encoded, setting as is") + } else { + errorType = string(decodedType) + } errorStack := r.Header.Get(invocationlifecycle.InvocationErrorStackHeader) if decodedStack, err := base64.StdEncoding.DecodeString(errorStack); err != nil { log.Debug("Could not decode error stack header") diff --git a/pkg/serverless/daemon/routes_test.go b/pkg/serverless/daemon/routes_test.go index 25231204af6c8..e630d2829bd61 100644 --- a/pkg/serverless/daemon/routes_test.go +++ b/pkg/serverless/daemon/routes_test.go @@ -7,6 +7,7 @@ package daemon import ( "bytes" + "encoding/base64" "fmt" "io" "net/http" @@ -104,7 +105,7 @@ func TestEndInvocation(t *testing.T) { assert.Equal(m.lastEndDetails.Runtime, d.ExecutionContext.GetCurrentState().Runtime) } -func TestEndInvocationWithError(t *testing.T) { +func TestEndInvocationWithErrorEncodedHeaders(t *testing.T) { assert := assert.New(t) port := testutil.FreeTCPPort(t) d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) @@ -114,10 +115,52 @@ func TestEndInvocationWithError(t *testing.T) { m := &mockLifecycleProcessor{} d.InvocationProcessor = m + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message \n at TestFunction.Handle(ILambdaContext context)" + + client := &http.Client{} + body := bytes.NewBuffer([]byte(`{}`)) + request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) + request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, base64.StdEncoding.EncodeToString([]byte(errorMessage))) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, base64.StdEncoding.EncodeToString([]byte(errorType))) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, base64.StdEncoding.EncodeToString([]byte(errorStack))) + assert.Nil(err) + res, err := client.Do(request) + assert.Nil(err) + if res != nil { + res.Body.Close() + assert.Equal(res.StatusCode, 200) + } + assert.True(m.OnInvokeEndCalled) + assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) +} + +func TestEndInvocationWithErrorNonEncodedHeaders(t *testing.T) { + assert := assert.New(t) + port := testutil.FreeTCPPort(t) + d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) + time.Sleep(100 * time.Millisecond) + defer d.Stop() + + m := &mockLifecycleProcessor{} + d.InvocationProcessor = m + + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message at TestFunction.Handle(ILambdaContext context)" + client := &http.Client{} body := bytes.NewBuffer([]byte(`{}`)) request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, errorMessage) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, errorType) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, errorStack) assert.Nil(err) res, err := client.Do(request) assert.Nil(err) @@ -127,6 +170,9 @@ func TestEndInvocationWithError(t *testing.T) { } assert.True(m.OnInvokeEndCalled) assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) } func TestTraceContext(t *testing.T) { From f803a52458da24266aa502d72d27dfdafdef6fcf Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 18 Dec 2024 21:38:42 +0100 Subject: [PATCH 22/26] Revert "bump github.com/DataDog/go-sqllexer to v0.0.18" (#32365) This PR broke a KMT test (pkg/network/protocols/postgres.TestExtractTableFunction/single_table_name_with_mixed_caps). #incident-33427 --- comp/otelcol/ddflareextension/impl/go.mod | 2 +- comp/otelcol/ddflareextension/impl/go.sum | 4 ++-- comp/otelcol/otlp/components/exporter/datadogexporter/go.mod | 2 +- comp/otelcol/otlp/components/exporter/datadogexporter/go.sum | 4 ++-- comp/otelcol/otlp/components/statsprocessor/go.mod | 2 +- comp/otelcol/otlp/components/statsprocessor/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- pkg/config/remote/go.mod | 2 +- pkg/config/remote/go.sum | 4 ++-- pkg/obfuscate/go.mod | 2 +- pkg/obfuscate/go.sum | 4 ++-- pkg/trace/go.mod | 2 +- pkg/trace/go.sum | 4 ++-- pkg/trace/stats/oteltest/go.mod | 2 +- pkg/trace/stats/oteltest/go.sum | 4 ++-- test/otel/go.mod | 2 +- test/otel/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index ce08a7999f072..21d3f17b9066a 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -267,7 +267,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index 7aed291a64674..d4f8e67e79da5 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -70,8 +70,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index dfe091b9d4b79..301ef0f3b1f48 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -201,7 +201,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index c32ab7009cbeb..e9c507fbafa8a 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index c03fc39e3ca59..18606500a04d2 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -43,7 +43,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index 22ef689018918..a19fb370fa6c4 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/go.mod b/go.mod index 959651ab0bcce..c8de7db9cc5f0 100644 --- a/go.mod +++ b/go.mod @@ -735,7 +735,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/DataDog/go-libddwaf/v3 v3.5.1 - github.com/DataDog/go-sqllexer v0.0.18 + github.com/DataDog/go-sqllexer v0.0.17 github.com/Datadog/dublin-traceroute v0.0.2 github.com/aquasecurity/trivy v0.49.2-0.20240227072422-e1ea02c7b80d github.com/aws/aws-sdk-go-v2/service/kms v1.37.6 diff --git a/go.sum b/go.sum index ffa368593f411..26375333ef0cf 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302 github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302-b9cf785c02fe/go.mod h1:90sqV0j7E8wYCyqIp5d9HmYWLTFQttqPFFtNYDyAybQ= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 72ed5685445f8..a1c4a4ebbc561 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -87,7 +87,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index 512cb6bacbab2..ca537b0d1660e 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -11,8 +11,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index 709579d414b7f..b34df591521b5 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/DataDog/datadog-go/v5 v5.6.0 - github.com/DataDog/go-sqllexer v0.0.18 + github.com/DataDog/go-sqllexer v0.0.17 github.com/outcaste-io/ristretto v0.2.3 github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index 940c4e5438e4a..063bd88a07005 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index c3f102a944494..29bf241afd7b8 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -61,7 +61,7 @@ require go.opentelemetry.io/collector/processor v0.115.0 // indirect require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/zstd v1.5.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index efd28e50f6d16..e1b7335487809 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 1259dbf55e8bf..071e7c2c2eba0 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -30,7 +30,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index 22ef689018918..a19fb370fa6c4 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/test/otel/go.mod b/test/otel/go.mod index beb5f560e099e..88d5f3a127be5 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -179,7 +179,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index 4ccec6f105aa4..bce88ff15bfa8 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.22.0 h1:r1Dx2cRHCBWkVluSZA41i4eoI/nOGbcrrZdkqWjoFCc= From 919ff87361c87fa18c5e7498565f26e4fe2271f9 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Wed, 18 Dec 2024 16:20:11 -0500 Subject: [PATCH 23/26] [OTEL-2125] Add configsync to otel-agent without DD exporter config (#32363) --- cmd/otel-agent/subcommands/run/command.go | 15 ++++++++++++++- comp/otelcol/ddflareextension/impl/server.go | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index a414674620e44..08c1472998157 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/core/hostname/remotehostnameimpl" log "github.com/DataDog/datadog-agent/comp/core/log/def" + logfx "github.com/DataDog/datadog-agent/comp/core/log/fx" logtracefx "github.com/DataDog/datadog-agent/comp/core/log/fx-trace" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" @@ -106,8 +107,20 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Provide(func() coreconfig.Component { return acfg }), + fx.Provide(func(_ coreconfig.Component) log.Params { + return log.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile) + }), + logfx.Module(), + fetchonlyimpl.Module(), + // TODO: don't rely on this pattern; remove this `OptionalModuleWithParams` thing + // and instead adapt OptionalModule to allow parameter passing naturally. + // See: https://github.com/DataDog/datadog-agent/pull/28386 + configsyncimpl.OptionalModuleWithParams(), + fx.Provide(func() configsyncimpl.Params { + return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) + }), converterfx.Module(), - fx.Provide(func(cp converter.Component) confmap.Converter { + fx.Provide(func(cp converter.Component, _ optional.Option[configsync.Component]) confmap.Converter { return cp }), collectorcontribFx.Module(), diff --git a/comp/otelcol/ddflareextension/impl/server.go b/comp/otelcol/ddflareextension/impl/server.go index ce1d2ceadef52..233dcddee1977 100644 --- a/comp/otelcol/ddflareextension/impl/server.go +++ b/comp/otelcol/ddflareextension/impl/server.go @@ -116,7 +116,7 @@ func newServer(endpoint string, handler http.Handler, auth bool) (*server, error // no easy way currently to pass required bearer auth token to OSS collector; // skip the validation if running inside a separate collector // TODO: determine way to allow OSS collector to authenticate with agent, OTEL-2226 - if auth { + if auth && util.GetAuthToken() != "" { r.Use(validateToken) } From f4ae8f0fd12ff1bc2c42f54f07dc00223f492a22 Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Wed, 18 Dec 2024 23:25:20 +0100 Subject: [PATCH 24/26] [CWS] turn fentry on by default (#31630) --- .../kernel_matrix_testing/security_agent.yml | 42 ------------------- pkg/config/setup/system_probe.go | 6 +-- .../test-runner/files/cws_fentry.json | 10 ----- 3 files changed, 3 insertions(+), 55 deletions(-) delete mode 100644 test/new-e2e/system-probe/test-runner/files/cws_fentry.json diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index b7f4b80cc6ed1..769ebe10138b8 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -216,26 +216,6 @@ kmt_run_secagent_tests_x64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_x64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - needs: - - kmt_setup_env_secagent_x64 - - upload_dependencies_secagent_x64 - - upload_secagent_tests_x64 - variables: - ARCH: "x86_64" - parallel: - matrix: - - TAG: - - "amazon_2023" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_x64_docker: extends: - .kmt_run_secagent_tests @@ -350,26 +330,6 @@ kmt_run_secagent_tests_arm64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_arm64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - needs: - - kmt_setup_env_secagent_arm64 - - upload_dependencies_secagent_arm64 - - upload_secagent_tests_arm64 - variables: - ARCH: "arm64" - parallel: - matrix: - - TAG: - - "ubuntu_24.04" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_arm64_docker: extends: - .kmt_run_secagent_tests @@ -426,7 +386,6 @@ kmt_secagent_tests_join_arm64: - kmt_run_secagent_tests_arm64 - kmt_run_secagent_tests_arm64_ad - kmt_run_secagent_tests_arm64_ebpfless - - kmt_run_secagent_tests_arm64_fentry - kmt_run_secagent_tests_arm64_docker kmt_secagent_cleanup_arm64: @@ -449,7 +408,6 @@ kmt_secagent_tests_join_x64: - kmt_run_secagent_tests_x64_required - kmt_run_secagent_tests_x64_ad - kmt_run_secagent_tests_x64_ebpfless - - kmt_run_secagent_tests_x64_fentry - kmt_run_secagent_tests_x64_docker kmt_secagent_cleanup_x64: diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index bdc97ee3d902a..1ca54550cec5a 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -371,9 +371,9 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.raw_classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_ring_buffer"), true) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_amd64"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_arm64"), false) + eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), true) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_amd64")) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_arm64")) eventMonitorBindEnv(cfg, join(evNS, "event_stream.buffer_size")) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "envs_with_value"), []string{"LD_PRELOAD", "LD_LIBRARY_PATH", "PATH", "HISTSIZE", "HISTFILESIZE", "GLIBC_TUNABLES"}) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "runtime_compilation.enabled"), false) diff --git a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json b/test/new-e2e/system-probe/test-runner/files/cws_fentry.json deleted file mode 100644 index 8330f6d6048af..0000000000000 --- a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "filters": { - "*": { - "exclude": false - } - }, - "additional_env_vars": [ - "DD_EVENT_MONITORING_CONFIG_EVENT_STREAM_USE_FENTRY=true" - ] -} From f51b26c3ad1c5d705ea360562c6376f05a741578 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Wed, 18 Dec 2024 17:44:10 -0500 Subject: [PATCH 25/26] [NPM-3665] Include semodule -l in agent flare (#32189) Co-authored-by: DeForest Richards <56796055+drichards-87@users.noreply.github.com> --- cmd/system-probe/api/debug/handlers_linux.go | 29 ++++++++++++++----- .../api/debug/handlers_nolinux.go | 6 ++++ cmd/system-probe/api/server.go | 1 + pkg/flare/archive_linux.go | 7 +++++ .../flare-semodule-list-883aecc886cd62ac.yaml | 11 +++++++ 5 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml diff --git a/cmd/system-probe/api/debug/handlers_linux.go b/cmd/system-probe/api/debug/handlers_linux.go index d2bd7dfbd5f48..07ba06c49354f 100644 --- a/cmd/system-probe/api/debug/handlers_linux.go +++ b/cmd/system-probe/api/debug/handlers_linux.go @@ -17,19 +17,18 @@ import ( "time" ) -// HandleSelinuxSestatus reports the output of sestatus as an http result -func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - cmd := exec.CommandContext(ctx, "sestatus") +// handleCommand runs commandName with the provided arguments and writes it to the HTTP response. +// If the command exits with a failure or doesn't exist in the PATH, it will still 200 but report the failure. +// Any other kind of error will 500. +func handleCommand(ctx context.Context, w http.ResponseWriter, commandName string, args ...string) { + cmd := exec.CommandContext(ctx, commandName, args...) output, err := cmd.CombinedOutput() var execError *exec.Error var exitErr *exec.ExitError if err != nil { - // don't 500 for ExitErrors etc, to report "normal" failures to the selinux_sestatus.log file + // don't 500 for ExitErrors etc, to report "normal" failures to the flare log file if !errors.As(err, &execError) && !errors.As(err, &exitErr) { w.WriteHeader(500) } @@ -39,3 +38,19 @@ func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { w.Write(output) } + +// HandleSelinuxSestatus reports the output of sestatus as an http result +func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "sestatus") +} + +// HandleSelinuxSemoduleList reports the output of semodule -l as an http result +func HandleSelinuxSemoduleList(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "semodule", "-l") +} diff --git a/cmd/system-probe/api/debug/handlers_nolinux.go b/cmd/system-probe/api/debug/handlers_nolinux.go index 1475d821c1e6e..246f4a3a7c78a 100644 --- a/cmd/system-probe/api/debug/handlers_nolinux.go +++ b/cmd/system-probe/api/debug/handlers_nolinux.go @@ -18,3 +18,9 @@ func HandleSelinuxSestatus(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) io.WriteString(w, "HandleSelinuxSestatus is not supported on this platform") } + +// HandleSelinuxSemoduleList is not supported +func HandleSelinuxSemoduleList(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(500) + io.WriteString(w, "HandleSelinuxSemoduleList is not supported on this platform") +} diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index d81007a0c8f0d..f4d9e85522d91 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -60,6 +60,7 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet if runtime.GOOS == "linux" { mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) mux.HandleFunc("/debug/selinux_sestatus", debug.HandleSelinuxSestatus) + mux.HandleFunc("/debug/selinux_semodule_list", debug.HandleSelinuxSemoduleList) } go func() { diff --git a/pkg/flare/archive_linux.go b/pkg/flare/archive_linux.go index dafe8bd41d1bc..9a3aea87a0ac0 100644 --- a/pkg/flare/archive_linux.go +++ b/pkg/flare/archive_linux.go @@ -39,6 +39,7 @@ func addSystemProbePlatformSpecificEntries(fb flaretypes.FlareBuilder) { _ = fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_host.log"), getSystemProbeConntrackHost) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "ebpf_btf_loader.log"), getSystemProbeBTFLoaderInfo) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_sestatus.log"), getSystemProbeSelinuxSestatus) + _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_semodule_list.log"), getSystemProbeSelinuxSemoduleList) } } @@ -155,3 +156,9 @@ func getSystemProbeSelinuxSestatus() ([]byte, error) { url := sysprobeclient.DebugURL("/selinux_sestatus") return getHTTPData(sysProbeClient, url) } + +func getSystemProbeSelinuxSemoduleList() ([]byte, error) { + sysProbeClient := sysprobeclient.Get(getSystemProbeSocketPath()) + url := sysprobeclient.DebugURL("/selinux_semodule_list") + return getHTTPData(sysProbeClient, url) +} diff --git a/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml new file mode 100644 index 0000000000000..2baa2dea73281 --- /dev/null +++ b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Added the output of ``semodule -l`` to the Agent flare; this information appears in ``system-probe/selinux_semodule_list.log``. From 8ccce414826464f35a5176194292f8ac43f740a0 Mon Sep 17 00:00:00 2001 From: sabrina lu Date: Wed, 18 Dec 2024 19:43:13 -0500 Subject: [PATCH 26/26] remove unused choco related files (#32310) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .github/CODEOWNERS | 1 - .gitlab-ci.yml | 3 +- .gitlab/.ci-linters.yml | 1 - .gitlab/choco_build/choco_build.yml | 46 ++-------- .gitlab/choco_deploy/choco_deploy.yml | 32 ------- .../powershell_script_deploy.yml | 4 +- ...ent-online.nuspec => datadog-agent.nuspec} | 2 +- .../offline/datadog-agent-offline.nuspec | 31 ------- .../offline/tools/chocolateyinstall.ps1 | 29 ------ .../online/tools/VERIFICATION.txt | 5 -- .../{offline => }/tools/VERIFICATION.txt | 0 .../{online => }/tools/chocolateyinstall.ps1 | 0 ...nline.nuspec => datadog-fips-agent.nuspec} | 0 .../{online => }/tools/VERIFICATION.txt | 0 .../{online => }/tools/chocolateyinstall.ps1 | 0 .../Generate-Chocolatey-Package.ps1 | 89 ++++++++----------- .../Publish-Chocolatey-Package.ps1 | 12 --- tasks/winbuildscripts/chocopush.bat | 11 --- 18 files changed, 47 insertions(+), 219 deletions(-) delete mode 100644 .gitlab/choco_deploy/choco_deploy.yml rename chocolatey/datadog-agent/{online/datadog-agent-online.nuspec => datadog-agent.nuspec} (97%) delete mode 100644 chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec delete mode 100644 chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 delete mode 100644 chocolatey/datadog-agent/online/tools/VERIFICATION.txt rename chocolatey/datadog-agent/{offline => }/tools/VERIFICATION.txt (100%) rename chocolatey/datadog-agent/{online => }/tools/chocolateyinstall.ps1 (100%) rename chocolatey/datadog-fips-agent/{online/datadog-fips-agent-online.nuspec => datadog-fips-agent.nuspec} (100%) rename chocolatey/datadog-fips-agent/{online => }/tools/VERIFICATION.txt (100%) rename chocolatey/datadog-fips-agent/{online => }/tools/chocolateyinstall.ps1 (100%) delete mode 100644 tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 delete mode 100644 tasks/winbuildscripts/chocopush.bat diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 50f8ff18a0c96..cb9a89ff310d8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -148,7 +148,6 @@ /.gitlab/powershell_script_deploy @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/choco_build/choco_build.yml @DataDog/agent-delivery @DataDog/windows-agent -/.gitlab/choco_deploy/choco_deploy.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/integration_test/windows.yml @DataDog/agent-devx-infra @DataDog/windows-agent diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2038aa4649f91..753715d308c60 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,6 @@ include: - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml - - .gitlab/choco_deploy/choco_deploy.yml - .gitlab/powershell_script_signing/powershell_script_signing.yml - .gitlab/powershell_script_deploy/powershell_script_deploy.yml - .gitlab/common/shared.yml @@ -86,7 +85,7 @@ stages: - deploy_dca - choco_and_install_script_build - trigger_release - - choco_and_install_script_deploy + - install_script_deploy - internal_image_deploy - e2e_deploy - install_script_testing diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml index af5acfa72b74a..ef85f659c27b6 100644 --- a/.gitlab/.ci-linters.yml +++ b/.gitlab/.ci-linters.yml @@ -83,7 +83,6 @@ job-owners: - new-e2e-unit-tests - ot_agent_deb-arm64-a7 - ot_agent_deb-x64-a7 - - publish_choco_7_x64 - publish_fakeintake - publish_fakeintake_latest - rc_kubernetes_deploy diff --git a/.gitlab/choco_build/choco_build.yml b/.gitlab/choco_build/choco_build.yml index 8ec5bac814d18..06ff3c6bb5d0d 100644 --- a/.gitlab/choco_build/choco_build.yml +++ b/.gitlab/choco_build/choco_build.yml @@ -2,34 +2,8 @@ # choco_build stage # Contains jobs which build the chocolatey Agent package. -# Not currently used in the pipeline. -windows_choco_offline_7_x64: - rules: - !reference [.manual] - stage: choco_and_install_script_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_msi_and_bosh_zip_x64-a7"] - variables: - ARCH: "x64" - script: - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - copy omnibus\pkg\*.msi .\chocolatey\datadog-agent\offline\tools\ - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e AWS_NETWORKING=true - registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod offline -Flavor $FLAVOR -InstallDeps 1" - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - copy build-out\*.nupkg omnibus\pkg - artifacts: - expire_in: 2 weeks - paths: - - omnibus/pkg - -# The online version of the choco job gets the msi package through the gitlab artifacts -.windows_choco_online_7_x64: +# The choco job gets the msi package through the gitlab artifacts +.windows_choco_7_x64: rules: !reference [.on_deploy_stable_or_beta_repo_branch] stage: choco_and_install_script_build @@ -39,11 +13,7 @@ windows_choco_offline_7_x64: script: - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - $ErrorActionPreference = "Stop" - - mkdir temp\ - - copy omnibus\pkg\*.msi temp\ - - if (Test-Path omnibus) { remove-item -recurse -force omnibus } - if (Test-Path build-out) { remove-item -recurse -force build-out } - - mkdir omnibus\pkg - > docker run --rm -v "$(Get-Location):c:\mnt" @@ -52,10 +22,8 @@ windows_choco_offline_7_x64: -e BUCKET_BRANCH="$BUCKET_BRANCH" -e AWS_NETWORKING=true registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod online -MSIDirectory c:\mnt\temp -Flavor $FLAVOR -InstallDeps 1" + powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -MSIDirectory c:\mnt\omnibus\pkg -Flavor $FLAVOR -InstallDeps 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - Remove-Item -Path "temp\" -Recurse -Force - - copy build-out\*.nupkg omnibus\pkg - $CopyNupkgToS3 = "$S3_CP_CMD --recursive --exclude '*' --include '*.nupkg' build-out $S3_RELEASE_ARTIFACTS_URI/choco/nupkg" - Invoke-Expression $CopyNupkgToS3 artifacts: @@ -65,8 +33,8 @@ windows_choco_offline_7_x64: # Sometimes Chocolatey is flakey retry: 2 -windows_choco_online_7_x64: - extends: .windows_choco_online_7_x64 +windows_choco_7_x64: + extends: .windows_choco_7_x64 # On dev/PR branches: # - if the job is run manually it will create a package, but before the # package can be installed, the deploy_windows_testing-a7 job must @@ -75,8 +43,8 @@ windows_choco_online_7_x64: variables: FLAVOR: "datadog-agent" -windows_choco_online_7_x64-fips: - extends: .windows_choco_online_7_x64 +windows_choco_7_x64-fips: + extends: .windows_choco_7_x64 # On dev/PR branches: # - if the job is run manually it will create a package, but before the # package can be installed, the deploy_windows_testing-a7-fips job must diff --git a/.gitlab/choco_deploy/choco_deploy.yml b/.gitlab/choco_deploy/choco_deploy.yml deleted file mode 100644 index 05b0f0d83eba6..0000000000000 --- a/.gitlab/choco_deploy/choco_deploy.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# choco_build stage -# Contains a job which deploys the chocolatey Agent package. - -publish_choco_7_x64: - rules: !reference [.on_deploy_stable_or_beta_repo_branch_manual] - stage: choco_and_install_script_deploy - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_choco_online_7_x64"] - variables: - ARCH: "x64" - before_script: - - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile") - - If ($lastExitCode -ne "0") { exit "$lastExitCode" } - - $chocolateyApiKey=$(cat "$tmpfile") - - Remove-Item "$tmpfile" - script: - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - if (Test-Path nupkg) { remove-item -recurse -force nupkg } - - mkdir nupkg - - copy omnibus\pkg\*.nupkg nupkg\ - - Get-ChildItem nupkg - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e CHOCOLATEY_API_KEY=${chocolateyApiKey} - registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\chocopush.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } diff --git a/.gitlab/powershell_script_deploy/powershell_script_deploy.yml b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml index 4760dfdf18030..5269045adde05 100644 --- a/.gitlab/powershell_script_deploy/powershell_script_deploy.yml +++ b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml @@ -5,7 +5,7 @@ powershell_script_deploy: image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - stage: choco_and_install_script_deploy + stage: install_script_deploy rules: !reference [.manual] needs: ["powershell_script_signing"] @@ -19,7 +19,7 @@ powershell_script_deploy: windows_bootstrapper_deploy: image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - stage: choco_and_install_script_deploy + stage: install_script_deploy rules: !reference [.manual] needs: ["windows-installer-amd64"] diff --git a/chocolatey/datadog-agent/online/datadog-agent-online.nuspec b/chocolatey/datadog-agent/datadog-agent.nuspec similarity index 97% rename from chocolatey/datadog-agent/online/datadog-agent-online.nuspec rename to chocolatey/datadog-agent/datadog-agent.nuspec index 73f96bfb83871..f3d9ec86ca694 100644 --- a/chocolatey/datadog-agent/online/datadog-agent-online.nuspec +++ b/chocolatey/datadog-agent/datadog-agent.nuspec @@ -1,4 +1,4 @@ - + datadog-agent diff --git a/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec b/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec deleted file mode 100644 index aedf9555a1c51..0000000000000 --- a/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec +++ /dev/null @@ -1,31 +0,0 @@ - - - - datadog-agent-offline - $package_version$ - https://github.com/DataDog/datadog-agent/tree/main/chocolatey - Datadog - Datadog Agent Offline Install - Datadog - https://github.com/DataDog/datadog-agent - https://datadog-prod.imgix.net/img/dd_logo_70x75.png - $copyright$ - https://raw.githubusercontent.com/DataDog/datadog-agent/main/LICENSE - true - https://docs.datadoghq.com - datadog agent monitoring admin - The Datadog Agent for Microsoft Windows - The Datadog Agent faithfully collects events and metrics and brings them to Datadog on your behalf so that you can do something useful with your monitoring and performance data. - -## Package settings - -You may set [custom settings](https://docs.datadoghq.com/agent/basic_agent_usage/windows/?tab=commandline#installation) to the Agent when installing by using the [`--installer-arguments` option of `choco install`](https://chocolatey.org/docs/getting-started#overriding-default-install-directory-or-other-advanced-install-concepts). - -For example, to set the API key you may run: -`choco install -ia="APIKEY=""YOUR_DATADOG_API_KEY""" datadog-agent-offline` - $release_notes$ - - - - - diff --git a/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 b/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 deleted file mode 100644 index 98d39c92654e0..0000000000000 --- a/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -$ErrorActionPreference = 'Stop'; - -$toolsDir = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)" -$nupkgs = Get-ChildItem $toolsDir\datadog-agent*.msi -if (($nupkgs | Measure-Object).Count -gt 1) { - Write-Host "More than 1 MSI installer exists - aborting" - exit -2 -} -$packageArgs = @{ - packageName = $env:ChocolateyPackageName - unzipLocation = $toolsDir - fileType = 'msi' - file = $nupkgs[0].FullName - softwareName = 'Datadog Agent' - silentArgs = "/qn /norestart /l*v `"$($env:TEMP)\$($packageName).$($env:chocolateyPackageVersion).MsiInstall.log`"" - validExitCodes= @(0, 3010, 1641) -} -Install-ChocolateyInstallPackage @packageArgs - -$installInfo = @" ---- -install_method: - tool: chocolatey - tool_version: chocolatey-$($env:CHOCOLATEY_VERSION) - installer_version: chocolatey_package-offline -"@ - -$appDataDir = (Get-ItemProperty -Path "HKLM:\SOFTWARE\Datadog\Datadog Agent").ConfigRoot -Out-File -FilePath $appDataDir\install_info -InputObject $installInfo diff --git a/chocolatey/datadog-agent/online/tools/VERIFICATION.txt b/chocolatey/datadog-agent/online/tools/VERIFICATION.txt deleted file mode 100644 index 5ef7f466e2cd6..0000000000000 --- a/chocolatey/datadog-agent/online/tools/VERIFICATION.txt +++ /dev/null @@ -1,5 +0,0 @@ -VERIFICATION -Verification is intended to assist the Chocolatey moderators and community in verifying that this package's contents are trustworthy. - -This package is published by Datadog itself. -The binaries are identical to other package types for the Datadog Agent. diff --git a/chocolatey/datadog-agent/offline/tools/VERIFICATION.txt b/chocolatey/datadog-agent/tools/VERIFICATION.txt similarity index 100% rename from chocolatey/datadog-agent/offline/tools/VERIFICATION.txt rename to chocolatey/datadog-agent/tools/VERIFICATION.txt diff --git a/chocolatey/datadog-agent/online/tools/chocolateyinstall.ps1 b/chocolatey/datadog-agent/tools/chocolateyinstall.ps1 similarity index 100% rename from chocolatey/datadog-agent/online/tools/chocolateyinstall.ps1 rename to chocolatey/datadog-agent/tools/chocolateyinstall.ps1 diff --git a/chocolatey/datadog-fips-agent/online/datadog-fips-agent-online.nuspec b/chocolatey/datadog-fips-agent/datadog-fips-agent.nuspec similarity index 100% rename from chocolatey/datadog-fips-agent/online/datadog-fips-agent-online.nuspec rename to chocolatey/datadog-fips-agent/datadog-fips-agent.nuspec diff --git a/chocolatey/datadog-fips-agent/online/tools/VERIFICATION.txt b/chocolatey/datadog-fips-agent/tools/VERIFICATION.txt similarity index 100% rename from chocolatey/datadog-fips-agent/online/tools/VERIFICATION.txt rename to chocolatey/datadog-fips-agent/tools/VERIFICATION.txt diff --git a/chocolatey/datadog-fips-agent/online/tools/chocolateyinstall.ps1 b/chocolatey/datadog-fips-agent/tools/chocolateyinstall.ps1 similarity index 100% rename from chocolatey/datadog-fips-agent/online/tools/chocolateyinstall.ps1 rename to chocolatey/datadog-fips-agent/tools/chocolateyinstall.ps1 diff --git a/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 b/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 index 991317602152f..41bf71748cc2b 100644 --- a/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 +++ b/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 @@ -2,11 +2,8 @@ .SYNOPSIS Generates a Chocolatey package for the Datadog Agent. -.PARAMETER installMethod -Specifies the installation method. Valid values are "offline" and "online". This parameter is mandatory. - .PARAMETER msiDirectory -Specifies the directory containing the MSI file that will be used to calculate the checksum. This parameter is mandatory when the installMethod is "online". +Specifies the directory containing the MSI file that will be used to calculate the checksum. .PARAMETER Flavor Specifies the flavor of the Datadog Agent. The default value is "datadog-agent". @@ -18,23 +15,18 @@ Overrides the Agent version when building packages locally for testing. Indicates whether to install dependencies. The default value is $true. .EXAMPLE -.\Generate-Chocolatey-Package.ps1 -installMethod online -Flavor datadog-agent -VersionOverride "7.62.0" -msiDirectory C:\mnt\omnibus\pkg\ +.\Generate-Chocolatey-Package.ps1 -Flavor datadog-agent -VersionOverride "7.62.0" -msiDirectory C:\mnt\omnibus\pkg Generates a chocolatey package for 7.62.0, requires the MSI file to be present in MSIDirectory. .EXAMPLE -$env:CI_PIPELINE_ID="50910739"; .\Generate-Chocolatey-Package.ps1 -installMethod online -Flavor datadog-agent -VersionOverride "7.62.0-devel.git.276.e59b1b3.pipeline.50910739" -msiDirectory C:\mnt\omnibus\pkg +$env:CI_PIPELINE_ID="50910739"; .\Generate-Chocolatey-Package.ps1 -Flavor datadog-agent -VersionOverride "7.62.0-devel.git.276.e59b1b3.pipeline.50910739" -msiDirectory C:\mnt\omnibus\pkg Generates a chocolatey package for PR/devel build 7.62.0-devel.git.276.e59b1b3.pipeline.50910739, requires the MSI file to be present in MSIDirectory. The generated chocolatey package requires the MSI be uploaded to the dd-agent-mstesting bucket. #> Param( [Parameter(Mandatory=$true)] - [ValidateSet("offline", "online")] - [String] - $installMethod, - - [Parameter(Mandatory=$false)] [String] $msiDirectory, @@ -69,9 +61,9 @@ if (![string]::IsNullOrEmpty($VersionOverride)) { } $copyright = "Datadog {0}" -f (Get-Date).Year -$releasePattern = "(\d+\.\d+\.\d+)" -$releaseCandidatePattern = "(\d+\.\d+\.\d+)-rc\.(\d+)" -$develPattern = "(\d+\.\d+\.\d+)-devel\.git\.\d+\.(.+)" +$releasePattern = "^(\d+\.\d+\.\d+)$" +$releaseCandidatePattern = "^(\d+\.\d+\.\d+)-rc\.(\d+)$" +$develPattern = "^(\d+\.\d+\.\d+)-devel\.git\.\d+\.(.+)" # Build the package in a temporary directory # Some of the build steps modify the package source, so we don't want to do this in the source directory @@ -87,21 +79,16 @@ try { # For historical reasons, use a different artifact name for the datadog-agent flavor # See agent-release-management for more details $artifactName = "ddagent-cli" - $packageSource = "$repoRoot\chocolatey\datadog-agent\$installMethod" - $nuspecFile = "datadog-agent-$installMethod.nuspec" } elseif ($Flavor -eq "datadog-fips-agent") { - if ($installMethod -eq "offline") { - Write-Error "Offline install method not supported for flavor $Flavor" - exit 1 - } $artifactName = "datadog-fips-agent" - $packageSource = "$repoRoot\chocolatey\datadog-fips-agent\online" - $nuspecFile = "datadog-fips-agent-online.nuspec" } else { Write-Error "Unknown flavor $Flavor" exit 1 } + $packageSource = "$repoRoot\chocolatey\$Flavor" + $nuspecFile = "$Flavor.nuspec" + # These files/directories are referenced in the nuspec file $licensePath = "tools\LICENSE.txt" $installScript = "tools\chocolateyinstall.ps1" @@ -118,24 +105,22 @@ try { $releaseNotes = "https://github.com/DataDog/datadog-agent/releases/tag/{0}-rc.{1}" -f $agentVersionMatches.Matches.Groups[1], $agentVersionMatches.Matches.Groups[2] $url = "https://s3.amazonaws.com/dd-agent-mstesting/builds/beta/$artifactName-$($agentVersionMatches.Matches.Groups[1])-rc.$($agentVersionMatches.Matches.Groups[2]).msi" } elseif ($rawAgentVersion -match $develPattern) { - if ($installMethod -eq "online") { - # For devel builds/branches, use the dd-agent-mstesting bucket URL - # This allows us to build and test the package in PRs, and locally - # by using the `-VersionOverride` param. - if ([string]::IsNullOrEmpty($env:CI_PIPELINE_ID)) { - Write-Error "CI_PIPELINE_ID is not set, aborting" - exit 1 - } else { - if ($rawAgentVersion -notmatch $env:CI_PIPELINE_ID) { - Write-Error "CI_PIPELINE_ID is not found in the agent version, aborting" -ErrorAction Continue - if ([string]::IsNullOrEmpty($env:BUCKET_BRANCH)) { - # inv agent.version requires BUCKET_BRANCH to be set when including pipeline in version - Write-Error "BUCKET_BRANCH is not set, if you are running this locally, set `$env:BUCKET_BRANCH='dev' or pass the -VersionOverride parameter" -ErrorAction Continue - } - exit 1 + # For devel builds/branches, use the dd-agent-mstesting bucket URL + # This allows us to build and test the package in PRs, and locally + # by using the `-VersionOverride` param. + if ([string]::IsNullOrEmpty($env:CI_PIPELINE_ID)) { + Write-Error "CI_PIPELINE_ID is not set, aborting" + exit 1 + } else { + if ($rawAgentVersion -notmatch $env:CI_PIPELINE_ID) { + Write-Error "CI_PIPELINE_ID is not found in the agent version, aborting" -ErrorAction Continue + if ([string]::IsNullOrEmpty($env:BUCKET_BRANCH)) { + # inv agent.version requires BUCKET_BRANCH to be set when including pipeline in version + Write-Error "BUCKET_BRANCH is not set, if you are running this locally, set `$env:BUCKET_BRANCH='dev' or pass the -VersionOverride parameter" -ErrorAction Continue } - $url = "https://s3.amazonaws.com/dd-agent-mstesting/pipelines/A7/$env:CI_PIPELINE_ID/$flavor-$rawAgentVersion-1-x86_64.msi" + exit 1 } + $url = "https://s3.amazonaws.com/dd-agent-mstesting/pipelines/A7/$env:CI_PIPELINE_ID/$flavor-$rawAgentVersion-1-x86_64.msi" } $agentVersionMatches = $rawAgentVersion | Select-String -Pattern $develPattern $agentVersion = "{0}-devel-{1}" -f $agentVersionMatches.Matches.Groups[1], $agentVersionMatches.Matches.Groups[2].Value @@ -151,26 +136,24 @@ try { exit 1 } - Write-Host "Generating Chocolatey $installMethod package $flavor version $agentVersion in $(Get-Location)" + Write-Host "Generating Chocolatey package $flavor version $agentVersion in $(Get-Location)" # Template the install script with the URL and checksum - if ($installMethod -eq "online") { - try { - $tempMsi = Join-Path -Path "$msiDirectory" "$flavor-$rawAgentVersion-1-x86_64.msi" - if (!(Test-Path $tempMsi)) { - Write-Host "Error: Could not find MSI file in $tempMsi" - Get-ChildItem "$msiDirectory" - exit 1 - } - $checksum = (Get-FileHash $tempMsi -Algorithm SHA256).Hash - } - catch { - Write-Host "Error: Could not generate checksum for package $($tempMsi): $($_)" + try { + $msiPath = Join-Path -Path "$msiDirectory" "$flavor-$rawAgentVersion-1-x86_64.msi" + if (!(Test-Path $msiPath)) { + Write-Host "Error: Could not find MSI file in $msiPath" + Get-ChildItem "$msiDirectory" exit 1 } - # Set the $url in the install script - (Get-Content $installScript).replace('$__url_from_ci__', '"' + $url + '"').replace('$__checksum_from_ci__', '"' + $checksum + '"') | Set-Content $installScript + $checksum = (Get-FileHash $msiPath -Algorithm SHA256).Hash + } + catch { + Write-Host "Error: Could not generate checksum for package $($msiPath): $($_)" + exit 1 } + # Set the $url in the install script + (Get-Content $installScript).replace('$__url_from_ci__', '"' + $url + '"').replace('$__checksum_from_ci__', '"' + $checksum + '"') | Set-Content $installScript Write-Host "Generated nuspec file:" Write-Host (Get-Content $installScript | Out-String) diff --git a/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 b/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 deleted file mode 100644 index e8879285da1ee..0000000000000 --- a/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -$ErrorActionPreference = 'Stop'; - -# Install chocolatey binary -$env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - -Set-Location c:\mnt -$nupkgs = Get-ChildItem .\nupkg\datadog-agent*.nupkg -foreach($nupkg in $nupkgs) { - Write-Host "Publishing Chocolatey package $($nupkg.Name) for agent version $agentVersion" - choco push $nupkg.FullName --verbose --key $env:CHOCOLATEY_API_KEY --source https://push.chocolatey.org/ - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } -} diff --git a/tasks/winbuildscripts/chocopush.bat b/tasks/winbuildscripts/chocopush.bat deleted file mode 100644 index d359e3c5c2dec..0000000000000 --- a/tasks/winbuildscripts/chocopush.bat +++ /dev/null @@ -1,11 +0,0 @@ -if not exist c:\mnt\ goto nomntdir - -@echo c:\mnt found, continuing - -Powershell -C "C:\mnt\tasks\winbuildscripts\Publish-Chocolatey-Package.ps1" || exit /b 1 -goto :EOF - -:nomntdir -@echo directory not mounted, parameters incorrect -exit /b 2 -goto :EOF \ No newline at end of file