diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cd9cafea971f8b..dea6adac9cedb3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -220,7 +220,7 @@ /cmd/otel-agent/ @DataDog/opentelemetry /cmd/process-agent/ @DataDog/processes /cmd/serverless/ @DataDog/serverless @Datadog/serverless-aws -/cmd/serverless/dependencies*.txt @DataDog/serverless @DataDog/agent-shared-components +/cmd/serverless/ @DataDog/serverless /cmd/serverless-init/ @DataDog/serverless /cmd/system-probe/ @DataDog/ebpf-platform /cmd/system-probe/config/adjust_npm.go @DataDog/ebpf-platform @DataDog/Networks diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index a1cab0d65cb6ec..32b47ef123999e 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -24,8 +24,6 @@ prepare_secagent_ebpf_functional_tests* @DataDog/agent-security # Send count metrics about Golang dependencies golang_deps_send_count_metrics @DataDog/agent-shared-components -# Golang test dependecies diff -golang_deps_test @DataDog/agent-shared-components # Golang dependency diff generation golang_deps_diff @DataDog/ebpf-platform golang_deps_commenter @DataDog/ebpf-platform diff --git a/.gitlab/source_test/golang_deps_diff.yml b/.gitlab/source_test/golang_deps_diff.yml index 40ec1b3595188f..3454f19a584761 100644 --- a/.gitlab/source_test/golang_deps_diff.yml +++ b/.gitlab/source_test/golang_deps_diff.yml @@ -66,15 +66,3 @@ golang_deps_send_count_metrics: # Get API key to send metrics - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv -e go-deps.send-count-metrics --git-sha "${CI_COMMIT_SHA}" --git-ref "${CI_COMMIT_REF_NAME}" - -golang_deps_test: - stage: source_test - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - rules: - - when: on_success - needs: ["go_deps"] - before_script: - - !reference [.retrieve_linux_go_deps] - script: - - inv -e go-deps.test-list diff --git a/cmd/serverless/dependency_list_test.go b/cmd/serverless/dependency_list_test.go new file mode 100644 index 00000000000000..abd0df4f11d995 --- /dev/null +++ b/cmd/serverless/dependency_list_test.go @@ -0,0 +1,57 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build test && linux + +package main + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const erroMsg = ` +The %s_dependencies_%s.txt file is out of date. +Update the file locally with this content: +%s +` + +func buildDependencyList() (string, error) { + run := "go" + arg0 := "list" + arg1 := "-f" + arg2 := `"{{ join .Deps "\n"}}"` + arg3 := "-tags" + arg4 := "serverless,otlp" + arg5 := "github.com/DataDog/datadog-agent/cmd/serverless" + cmd := exec.Command(run, arg0, arg1, arg2, arg3, arg4, arg5) + + stdout, err := cmd.Output() + return string(stdout), err +} + +// This test is here to add friction to the process of adding dependencies to the serverless binary. +// If you are adding a dependency to the serverless binary, you must update the dependencies.txt file. +// Same for when you remove a dependency. +// Having this test also allow us to better track additions and removals of dependencies for the serverless binary. +func TestImportPackage(t *testing.T) { + dependencyList, err := buildDependencyList() + assert.NoError(t, err) + file := fmt.Sprintf("%s_dependencies_%s.txt", runtime.GOOS, runtime.GOARCH) + + data, err := os.ReadFile(file) + assert.NoError(t, err) + + cleanDependencyList := strings.TrimLeft(dependencyList, "\"") + cleanDependencyList = strings.TrimRight(cleanDependencyList, "\"\n") + cleanDependencyList += "\n" + assert.Equal(t, string(data), cleanDependencyList, fmt.Sprintf(erroMsg, runtime.GOOS, runtime.GOARCH, cleanDependencyList)) +} diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/linux_dependencies_amd64.txt similarity index 99% rename from cmd/serverless/dependencies_linux_amd64.txt rename to cmd/serverless/linux_dependencies_amd64.txt index 17a03e9a59a981..9f6caaf97d352a 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/linux_dependencies_amd64.txt @@ -1121,4 +1121,4 @@ vendor/golang.org/x/sys/cpu vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm \ No newline at end of file +vendor/golang.org/x/text/unicode/norm diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/linux_dependencies_arm64.txt similarity index 99% rename from cmd/serverless/dependencies_linux_arm64.txt rename to cmd/serverless/linux_dependencies_arm64.txt index eb97a914995a98..2d10f2a12b056d 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/linux_dependencies_arm64.txt @@ -1119,4 +1119,4 @@ vendor/golang.org/x/net/idna vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm \ No newline at end of file +vendor/golang.org/x/text/unicode/norm diff --git a/tasks/go_deps.py b/tasks/go_deps.py index d871734cecc558..b7bcbc25fd1ad8 100644 --- a/tasks/go_deps.py +++ b/tasks/go_deps.py @@ -1,7 +1,5 @@ import datetime -import io import os -from collections import namedtuple from collections.abc import Iterable from invoke.context import Context @@ -92,33 +90,6 @@ def compute_all_count_metrics(ctx: Context, extra_tags: Iterable[str] = ()): return series -def compute_binary_dependencies_list( - ctx: Context, - build: str, - flavor: AgentFlavor, - platform: str, - arch: str, -) -> list[str]: - """ - Compute binary import list for the given build/flavor/platform/arch. - """ - goos, goarch = GOOS_MAPPING[platform], GOARCH_MAPPING[arch] - - build_tags = get_default_build_tags(build=build, flavor=flavor, platform=platform) - - env = {"GOOS": goos, "GOARCH": goarch, "CGO_ENABLED": "1"} - cmd = "go list -f '{{ join .Deps \"\\n\"}}'" - - res = ctx.run( - f"{cmd} -tags {','.join(build_tags)}", - env=env, - hide='out', # don't hide errors - ) - assert res - - return [dep for dep in res.stdout.strip().splitlines() if not dep.startswith("internal/")] - - @task def send_count_metrics( ctx: Context, @@ -148,148 +119,3 @@ def send_count_metrics( print(color_message("Sending metrics to Datadog", "blue")) send_metrics(series=series) print(color_message("Done", "green")) - - -BINARY_TO_TEST = ["serverless"] -MisMatchBinary = namedtuple('MisMatchBinary', ['binary', 'os', 'arch', 'differences']) - - -@task -def test_list( - ctx: Context, -): - """ - Compare the dependencies list for the binaries in BINARY_TO_TEST with the actual dependencies of the binaries. - If the lists do not match, the task will raise an error. - """ - mismatch_binaries = set() - - for binary in BINARY_TO_TEST: - binary_info = BINARIES[binary] - entrypoint = binary_info["entrypoint"] - platforms = binary_info["platforms"] - flavor = binary_info.get("flavor", AgentFlavor.base) - build = binary_info.get("build", binary) - - with ctx.cd(entrypoint): - for platform in platforms: - platform, arch = platform.split("/") - - goos, goarch = GOOS_MAPPING[platform], GOARCH_MAPPING[arch] - - filename = os.path.join(ctx.cwd, f"dependencies_{goos}_{goarch}.txt") - if not os.path.isfile(filename): - print( - f"File {filename} does not exist. To execute the dependencies list check for the {binary} binary, please run the task `inv -e go-deps.generate" - ) - continue - - deps_file = open(filename) - deps = deps_file.read().strip().splitlines() - deps_file.close() - - list = compute_binary_dependencies_list(ctx, build, flavor, platform, arch) - - if list != deps: - new_dependencies_lines = len(list) - recorded_dependencies_lines = len(deps) - - mismatch_binaries.add( - MisMatchBinary(binary, goos, goarch, new_dependencies_lines - recorded_dependencies_lines) - ) - - if len(mismatch_binaries) > 0: - message = io.StringIO() - - for mismatch_binary in mismatch_binaries: - if mismatch_binary.differences > 0: - message.write( - color_message( - f"You added some dependencies to {mismatch_binary.binary} ({mismatch_binary.os}/{mismatch_binary.arch}). Adding new dependencies to the binary increases its size. Do we really need to add this dependency?\n", - "red", - ) - ) - else: - message.write( - color_message( - f"You removed some dependencies from {mismatch_binary.binary} ({mismatch_binary.os}/{mismatch_binary.arch}). Congratulations!\n", - "green", - ) - ) - - message.write( - color_message( - "To fix this check, please run `inv -e go-deps.generate`", - "orange", - ) - ) - - raise Exit( - code=1, - message=message.getvalue(), - ) - - -@task -def generate( - ctx: Context, -): - for binary in BINARY_TO_TEST: - binary_info = BINARIES[binary] - entrypoint = binary_info["entrypoint"] - platforms = binary_info["platforms"] - flavor = binary_info.get("flavor", AgentFlavor.base) - build = binary_info.get("build", binary) - - with ctx.cd(entrypoint): - for platform in platforms: - platform, arch = platform.split("/") - - goos, goarch = GOOS_MAPPING[platform], GOARCH_MAPPING[arch] - - filename = os.path.join(ctx.cwd, f"dependencies_{goos}_{goarch}.txt") - - list = compute_binary_dependencies_list(ctx, build, flavor, platform, arch) - - f = open(filename, "w") - f.write('\n'.join(list)) - f.close() - - -def key_for_value(map: dict[str, str], value: str) -> str: - """Return the key from a value in a dictionary.""" - for k, v in map.items(): - if v == value: - return k - raise ValueError(f"Unknown value {value}") - - -@task( - help={ - 'build': f'The agent build to use, one of {", ".join(BINARIES.keys())}', - 'flavor': f'The agent flavor to use, one of {", ".join(AgentFlavor.__members__.keys())}. Defaults to base', - 'os': f'The OS to use, one of {", ".join(GOOS_MAPPING.keys())}. Defaults to host platform', - 'arch': f'The architecture to use, one of {", ".join(GOARCH_MAPPING.keys())}. Defaults to host architecture', - } -) -def show(ctx: Context, build: str, flavor: str = AgentFlavor.base.name, os: str | None = None, arch: str | None = None): - """ - Print the Go dependency list for the given agent build/flavor/os/arch. - """ - - if os is None: - goos = ctx.run("go env GOOS", hide=True) - assert goos - os = key_for_value(GOOS_MAPPING, goos.stdout.strip()) - - if arch is None: - goarch = ctx.run("go env GOARCH", hide=True) - assert goarch - arch = key_for_value(GOARCH_MAPPING, goarch.stdout.strip()) - - entrypoint = BINARIES[build]["entrypoint"] - with ctx.cd(entrypoint): - deps = compute_binary_dependencies_list(ctx, build, AgentFlavor[flavor], os, arch) - - for dep in deps: - print(dep)