[EBPF] Backport KMT platform files to 6.53.x (#31699)

Co-authored-by: agent-platform-auto-pr[bot] <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> Co-authored-by: Bryce Kahle <[email protected]> Co-authored-by: Usama Saqib <[email protected]>
DataDog · Dec 18, 2024 · 36a2a3a · 36a2a3a
1 parent 57cbcf8
commit 36a2a3a
Show file tree

Hide file tree

Showing 16 changed files with 750 additions and 151 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -135,6 +135,7 @@ variables:
   S3_CP_OPTIONS: --only-show-errors --region us-east-1 --sse AES256
   S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS
   S3_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME/$CI_PIPELINE_ID
+  S3_PROJECT_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME
   S3_PERMANENT_ARTIFACTS_URI: s3://dd-ci-persistent-artefacts-build-stable/$CI_PROJECT_NAME
   S3_SBOM_STORAGE_URI: s3://sbom-root-us1-ddbuild-io/$CI_PROJECT_NAME/$CI_PIPELINE_ID
   S3_RELEASE_ARTIFACTS_URI: s3://dd-release-artifacts/$CI_PROJECT_NAME/$CI_PIPELINE_ID
@@ -146,7 +147,7 @@ variables:
   INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
   S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm
   S3_DD_AGENT_OMNIBUS_BTFS_URI: s3://dd-agent-omnibus/btfs
-  BTFHUB_ARCHIVE_BRANCH: no-kmod
+  BTFHUB_ARCHIVE_BRANCH: main
   GENERAL_ARTIFACTS_CACHE_BUCKET_URL: https://dd-agent-omnibus.s3.amazonaws.com
   S3_DSD6_URI: s3://dsd6-staging
   RELEASE_VERSION_6: nightly
@@ -1126,4 +1127,3 @@ workflow:
         - .gitlab-ci.yml
         - .gitlab/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
-
diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml
@@ -23,6 +23,7 @@
     expire_in: 2 weeks
     paths:
       - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
+      - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz.sum
 
 build_system-probe-x64:
   stage: binary_build

diff --git a/.gitlab/deps_build/deps_build.yml b/.gitlab/deps_build/deps_build.yml
@@ -90,5 +90,5 @@ build_processed_btfhub_archive:
     KUBERNETES_CPU_REQUEST: 32
   script:
     - inv -e system-probe.process-btfhub-archive --branch $BTFHUB_ARCHIVE_BRANCH
-    - $S3_CP_CMD btfs-x86_64.tar.gz $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-x86_64.tar.gz --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
-    - $S3_CP_CMD btfs-arm64.tar.gz $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-arm64.tar.gz --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+    - $S3_CP_CMD btfs-x86_64.tar $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-x86_64.tar --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+    - $S3_CP_CMD btfs-arm64.tar $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-arm64.tar --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml
@@ -216,9 +216,9 @@
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
     - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
     - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt"
-    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd '/root/fetch_dependencies.sh ${ARCH} && /opt/kernel-version-testing/micro-vm-init.sh -retry ${RETRY} -test-root /system-probe-tests -packages-run-config /${TEST_SET}.json'"
-    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
-    - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
+    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'CI=true /root/fetch_dependencies.sh ${ARCH} && /opt/kernel-version-testing/micro-vm-init.sh -retry ${RETRY} -test-root /opt/kmt-ramfs/${TEST_COMPONENT}-tests -packages-run-config /${TEST_SET}.json'"
+    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}" -send-env-vars DD_API_KEY # Allow DD_API_KEY to be passed to the metal instance, so we can use it to send metrics from the connector.
+    - ssh metal_instance "ssh ${MICRO_VM_IP} /test-json-review"
   artifacts:
     expire_in: 2 weeks
     when: always

diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml
@@ -147,8 +147,8 @@ kmt_run_secagent_tests_x64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_5.4"
-          - "amzn_5.10"
+          - "amazon_5.4"
+          - "amazon_5.10"
           - "fedora_37"
           - "fedora_38"
           - "debian_10"
@@ -181,8 +181,8 @@ kmt_run_secagent_tests_arm64:
       - TAG:
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_5.4"
-          - "amzn_5.10"
+          - "amazon_5.4"
+          - "amazon_5.10"
           - "fedora_37"
           - "fedora_38"
           - "debian_11"

diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml
@@ -5,7 +5,7 @@ upload_dependencies_sysprobe_x64:
   needs: ["pull_test_dockers_x64"]
   rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
   variables:
-    ARCH: amd64
+    ARCH: x86_64
     INSTANCE_TYPE: m5d.metal
     TEST_COMPONENT: system-probe
 
@@ -43,7 +43,7 @@ upload_dependencies_sysprobe_arm64:
 pull_test_dockers_x64:
   extends: .pull_test_dockers
   variables:
-    ARCH: amd64
+    ARCH: x86_64
 
 pull_test_dockers_arm64:
   extends: .pull_test_dockers
@@ -225,15 +225,15 @@ kmt_run_sysprobe_tests_x64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_4.14"
-          - "amzn_5.4"
-          - "amzn_5.10"
+          - "amazon_4.14"
+          - "amazon_5.4"
+          - "amazon_5.10"
           - "fedora_37"
           - "fedora_38"
           - "debian_10"
           - "debian_11"
           - "debian_12"
-          - "centos_79"
+          - "centos_7.9"
           - "centos_8"
         TEST_SET: ["no_tracersuite", "only_tracersuite"]
   after_script:
@@ -260,15 +260,15 @@ kmt_run_sysprobe_tests_arm64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_4.14"
-          - "amzn_5.4"
-          - "amzn_5.10"
+          - "amazon_4.14"
+          - "amazon_5.4"
+          - "amazon_5.10"
           - "fedora_37"
           - "fedora_38"
           - "debian_10"
           - "debian_11"
           - "debian_12"
-          - "centos_79"
+          - "centos_7.9"
           - "centos_8"
         TEST_SET: ["no_tracersuite", "only_tracersuite"]
   after_script:
@@ -282,6 +282,7 @@ kmt_run_sysprobe_tests_arm64:
     TEST_COMPONENT: system-probe
 
 kmt_sysprobe_cleanup_arm64:
+  when: always
   extends:
     - .kmt_sysprobe_cleanup
   needs:
@@ -292,6 +293,7 @@ kmt_sysprobe_cleanup_arm64:
     INSTANCE_TYPE: "m6gd.metal"
 
 kmt_sysprobe_cleanup_x64:
+  when: always
   extends:
     - .kmt_sysprobe_cleanup
   needs:

diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml
@@ -11,15 +11,30 @@
   tags: ["arch:amd64"]
   script:
     - cd $CI_PROJECT_DIR
-    - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar.gz .
-    - tar -xf btfs-$ARCH.tar.gz
+    - export BTFS_ETAG=$(aws s3api head-object --region us-east-1 --bucket dd-agent-omnibus --key btfs/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar --query ETag --output text | tr -d \")
+    - export OUTPUTS_HASH=$(sha256sum sysprobe-build-outputs.tar.xz.sum | cut -d' ' -f1)
+    - export MIN_BTFS_FILENAME=minimized-btfs-$BTFS_ETAG-$OUTPUTS_HASH.tar.xz
+    - |
+      # if running all builds, or this is a release branch, skip the cache check
+      if [[ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]]; then
+        if aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME; then
+          $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME $CI_PROJECT_DIR/minimized-btfs.tar.xz
+          echo "cached minimized BTFs exist"
+          exit 0
+        fi
+      fi
+    # cache does not exist, download processed BTFs and minimize
+    - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar .
+    - tar -xf btfs-$ARCH.tar
     - tar -xf sysprobe-build-outputs.tar.xz
     - inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --input-bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re"
     - cd minimized-btfs
     - tar -cJf $CI_PROJECT_DIR/minimized-btfs.tar.xz *
+    - $S3_CP_CMD $CI_PROJECT_DIR/minimized-btfs.tar.xz $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME
   variables:
-    KUBERNETES_MEMORY_REQUEST: "6Gi"
-    KUBERNETES_MEMORY_LIMIT: "12Gi"
+    KUBERNETES_MEMORY_REQUEST: "64Gi"
+    KUBERNETES_MEMORY_LIMIT: "64Gi"
+    KUBERNETES_CPU_REQUEST: 24
   artifacts:
     expire_in: 2 weeks
     paths:

diff --git a/tasks/kernel_matrix_testing/download.py b/tasks/kernel_matrix_testing/download.py
@@ -3,7 +3,7 @@
 import os
 import platform
 import tempfile
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
 
 from invoke.context import Context
 
@@ -18,7 +18,7 @@
     requests = None
 
 if TYPE_CHECKING:
-    from tasks.kernel_matrix_testing.types import PathOrStr
+    from tasks.kernel_matrix_testing.types import Arch, PathOrStr
 
 
 def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: str):
@@ -41,23 +41,27 @@ def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: st
     return False
 
 
-def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str):
+def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str, arch: Arch | None = None):
     platforms = get_platforms()
     vmconfig_template = get_vmconfig_template(vmconfig_template_name)
 
     url_base = platforms["url_base"]
 
-    arch = arch_mapping[platform.machine()]
-    to_download: List[str] = list()
-    file_ls: List[str] = list()
+    if arch is None:
+        arch = arch_mapping[platform.machine()]
+    to_download: list[str] = list()
+    file_ls: list[str] = list()
     branch_mapping: dict[str, str] = dict()
 
     for tag in platforms[arch]:
-        path = os.path.basename(platforms[arch][tag])
+        platinfo = platforms[arch][tag]
+        if "image" not in platinfo:
+            raise Exit("image is not defined in platform info")
+        path = os.path.basename(platinfo["image"])
         if path.endswith(".xz"):
             path = path[: -len(".xz")]
 
-        branch_mapping[path] = os.path.dirname(platforms[arch][tag]) or "master"
+        branch_mapping[path] = platinfo.get('image_version', 'master')
         file_ls.append(os.path.basename(path))
 
     # if file does not exist download it.
@@ -134,7 +138,11 @@ def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name:
         raise Exit("Failed to set permissions 0766 to rootfs")
 
 
-def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str):
-    download_rootfs(ctx, rootfs_dir, vmconfig_template)
+def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str, all_archs: bool = False):
+    if all_archs:
+        arch_ls: list[Arch] = ["x86_64", "arm64"]
+        for arch in arch_ls:
+            info(f"[+] Updating root filesystem for {arch}")
+            download_rootfs(ctx, rootfs_dir, vmconfig_template, arch)
 
     info("[+] Root filesystem and bootables images updated")
diff --git a/tasks/kernel_matrix_testing/types.py b/tasks/kernel_matrix_testing/types.py
@@ -25,10 +25,20 @@ class DependenciesLayout(TypedDict):  # noqa: F841
     build: Dict[str, DependencyBuild]
 
 
+class PlatformInfo(TypedDict, total=False):
+    os_name: str  # Official OS name  # noqa: F841
+    os_version: str  # Version  # noqa: F841
+    image_version: str  # Image version  # noqa: F841
+    kernel: str  # Kernel version
+    os_id: str  # Short ID for the OS (e.g., "centos" for CentOS)  # noqa: F841
+    image: str  # Name of the image file
+    alt_version_names: List[str]  # Alternative version names (e.g., "jammy" for Ubuntu 22)  # noqa: F841
+
+
 class Platforms(TypedDict):  # noqa: F841
     url_base: str
-    x86_64: Dict[str, str]  # noqa: F841
-    arm64: Dict[str, str]  # noqa: F841
+    x86_64: Dict[str, PlatformInfo]  # noqa: F841
+    arm64: Dict[str, PlatformInfo]  # noqa: F841
 
 
 class Disk(TypedDict):

diff --git a/tasks/kernel_matrix_testing/vars.py b/tasks/kernel_matrix_testing/vars.py
@@ -14,5 +14,6 @@
     "arm": "arm64",
     "aarch64": "arm64",
 }
+arch_ls: list[Arch] = ["x86_64", "arm64"]
 
 VMCONFIG = "vmconfig.json"