[ASCII-1796] Move list dependency test to python (#25866)

Co-authored-by: pgimalac <[email protected]>

.github/CODEOWNERS

@@ -176,6 +176,7 @@
 /cmd/otel-agent/                        @DataDog/opentelemetry
 /cmd/process-agent/                     @DataDog/processes
 /cmd/serverless/                        @DataDog/serverless
+/cmd/serverless/dependencies*.txt       @DataDog/serverless @DataDog/agent-shared-components
 /cmd/serverless-init/                   @DataDog/serverless
 /cmd/system-probe/                      @DataDog/ebpf-platform
 /cmd/system-probe/config/adjust_npm.go  @DataDog/ebpf-platform @DataDog/Networks

.gitlab/JOBOWNERS

@@ -15,6 +15,8 @@ prepare_secagent_ebpf_functional_tests* @DataDog/agent-security
 
 # Send count metrics about Golang dependencies
 golang_deps_send_count_metrics       @DataDog/agent-shared-components
+# Golang test dependecies diff
+golang_deps_test                     @DataDog/agent-shared-components
 # Golang dependency diff generation
 golang_deps_diff                     @DataDog/ebpf-platform
 golang_deps_commenter                @DataDog/ebpf-platform

.gitlab/source_test/golang_deps_diff.yml

@@ -62,3 +62,16 @@ golang_deps_send_count_metrics:
     # Get API key to send metrics
     - export DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $API_KEY_ORG2_SSM_NAME)
     - inv -e go-deps.send-count-metrics --git-sha "${CI_COMMIT_SHA}" --git-ref "${CI_COMMIT_REF_NAME}"
+
+golang_deps_test:
+  stage: source_test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  rules:
+    - when: on_success
+  needs: ["go_deps"]
+  before_script:
+    - source /root/.bashrc
+    - !reference [.retrieve_linux_go_deps]
+  script:
+    - inv -e go-deps.test-list

cmd/serverless/linux_dependencies_amd64.txt → cmd/serverless/dependencies_linux_amd64.txt

@@ -1052,4 +1052,4 @@ vendor/golang.org/x/sys/cpu
 vendor/golang.org/x/text/secure/bidirule
 vendor/golang.org/x/text/transform
 vendor/golang.org/x/text/unicode/bidi
-vendor/golang.org/x/text/unicode/norm
+vendor/golang.org/x/text/unicode/norm

cmd/serverless/linux_dependencies_arm64.txt → cmd/serverless/dependencies_linux_arm64.txt

@@ -1050,4 +1050,4 @@ vendor/golang.org/x/net/idna
 vendor/golang.org/x/text/secure/bidirule
 vendor/golang.org/x/text/transform
 vendor/golang.org/x/text/unicode/bidi
-vendor/golang.org/x/text/unicode/norm
+vendor/golang.org/x/text/unicode/norm

tasks/go_deps.py

@@ -1,5 +1,7 @@
 import datetime
+import io
 import os
+from collections import namedtuple
 from collections.abc import Iterable
 
 from invoke.context import Context
@@ -90,6 +92,33 @@ def compute_all_count_metrics(ctx: Context, extra_tags: Iterable[str] = ()):
     return series
 
 
+def compute_binary_dependencies_list(
+    ctx: Context,
+    build: str,
+    flavor: AgentFlavor,
+    platform: str,
+    arch: str,
+):
+    """
+    Compute binary import list for the given build/flavor/platform/arch.
+    """
+    goos, goarch = GOOS_MAPPING[platform], GOARCH_MAPPING[arch]
+
+    build_tags = get_default_build_tags(build=build, flavor=flavor, platform=platform)
+
+    env = {"GOOS": goos, "GOARCH": goarch, "CGO_ENABLED": "1"}
+    cmd = "go list -f '{{ join .Deps \"\\n\"}}'"
+
+    res = ctx.run(
+        f"{cmd} -tags {','.join(build_tags)}",
+        env=env,
+        hide='out',  # don't hide errors
+    )
+    assert res
+
+    return res.stdout.strip()
+
+
 @task
 def send_count_metrics(
     ctx: Context,
@@ -119,3 +148,109 @@ def send_count_metrics(
         print(color_message("Sending metrics to Datadog", "blue"))
         send_metrics(series=series)
         print(color_message("Done", "green"))
+
+
+BINARY_TO_TEST = ["serverless"]
+MisMacthBinary = namedtuple('failedBinary', ['binary', 'os', 'arch', 'differences'])
+
+
+@task
+def test_list(
+    ctx: Context,
+):
+    """
+    Compare the dependencies list for the binaries in BINARY_TO_TEST with the actual dependencies of the binaries.
+    If the lists do not match, the task will raise an error.
+    """
+    mismatch_binaries = set()
+
+    for binary in BINARY_TO_TEST:
+        binary_info = BINARIES[binary]
+        entrypoint = binary_info["entrypoint"]
+        platforms = binary_info["platforms"]
+        flavor = binary_info.get("flavor", AgentFlavor.base)
+        build = binary_info.get("build", binary)
+
+        with ctx.cd(entrypoint):
+            for platform in platforms:
+                platform, arch = platform.split("/")
+
+                goos, goarch = GOOS_MAPPING[platform], GOARCH_MAPPING[arch]
+
+                filename = os.path.join(ctx.cwd, f"dependencies_{goos}_{goarch}.txt")
+                if not os.path.isfile(filename):
+                    print(
+                        f"File {filename} does not exist. To execute the dependencies list check for the {binary} binary, please run the task `inv -e go-deps.generate --binaries {binary}"
+                    )
+                    continue
+
+                deps_file = open(filename)
+                deps = deps_file.read()
+                deps_file.close()
+
+                list = compute_binary_dependencies_list(ctx, build, flavor, platform, arch)
+
+                if list != deps:
+                    new_dependencies_lines = len(list.splitlines())
+                    recorded_dependencies_lines = len(deps.splitlines())
+
+                    mismatch_binaries.add(
+                        MisMacthBinary(binary, goos, goarch, new_dependencies_lines - recorded_dependencies_lines)
+                    )
+
+    if len(mismatch_binaries) > 0:
+        message = io.StringIO()
+
+        for mismatch_binary in mismatch_binaries:
+            if mismatch_binary.differences > 0:
+                message.write(
+                    color_message(
+                        f"You added some dependencies to {mismatch_binary.binary} ({mismatch_binary.os}/{mismatch_binary.arch}). Adding new dependencies to the binary increases its size. Do we really need to add this dependency?\n",
+                        "red",
+                    )
+                )
+            else:
+                message.write(
+                    color_message(
+                        f"You removed some dependencies from {mismatch_binary.binary} ({mismatch_binary.os}/{mismatch_binary.arch}). Congratulations!\n",
+                        "green",
+                    )
+                )
+
+        message.write(
+            color_message(
+                "To fix this check, please run `inv -e go-deps.generate`",
+                "orange",
+            )
+        )
+
+        raise Exit(
+            code=1,
+            message=message.getvalue(),
+        )
+
+
+@task
+def generate(
+    ctx: Context,
+):
+    for binary in BINARY_TO_TEST:
+        binary_info = BINARIES[binary]
+        entrypoint = binary_info["entrypoint"]
+        platforms = binary_info["platforms"]
+        flavor = binary_info.get("flavor", AgentFlavor.base)
+        build = binary_info.get("build", binary)
+
+        with ctx.cd(entrypoint):
+            for platform in platforms:
+                platform, arch = platform.split("/")
+
+                goos, goarch = GOOS_MAPPING[platform], GOARCH_MAPPING[arch]
+
+                filename = os.path.join(ctx.cwd, f"dependencies_{goos}_{goarch}.txt")
+
+                list = compute_binary_dependencies_list(ctx, build, flavor, platform, arch)
+
+                f = open(filename, "w")
+                f.write(list)
+                f.close()