[EBPF-474] Improve kernel information in platforms.json (#24097)

* Allow downloading images for all architectures

* Add update-platform-info task

* Improve output of kmt.ls

* Adapt to new platforms.json structure

* Use guestfs python package

* Ignore rescue images when detecting kernel

* Ignore empty version names

* Fix kernel detection

* Use dynamic distribution mappings

* Update platform info

* Remove empty alternative names

* Better help

* Fix archs for custom kernels

* Fix import order

* Fix format

* Use manifest files

* Improve sorting in kmt.ls

* Update platforms.json

* Warn when some images have not been updated

* Update platforms.json

* Update to latest images

* Raise an error when image is missing

* Fix platforms.json file

* Add kmt.validate-platform-info task

* review comments

* Improve kmt.ls

* Show an error when kernels are missing

* Fix image names in CI

* Switch Amazon 2023 version

* Use master version for AL2023

* test with updated names

* Update Amazon 2023 name

* Add --exclude-matching argument to update-platform-info task

* Update platforms.json

.gitlab/kernel_matrix_testing/security_agent.yml

@@ -147,8 +147,8 @@ kmt_run_secagent_tests_x64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_5.4"
-          - "amzn_5.10"
+          - "amazon_5.4"
+          - "amazon_5.10"
           - "fedora_37"
           - "fedora_38"
           - "debian_10"
@@ -181,8 +181,8 @@ kmt_run_secagent_tests_arm64:
       - TAG:
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_5.4"
-          - "amzn_5.10"
+          - "amazon_5.4"
+          - "amazon_5.10"
           - "fedora_37"
           - "fedora_38"
           - "debian_11"

.gitlab/kernel_matrix_testing/system_probe.yml

@@ -225,16 +225,16 @@ kmt_run_sysprobe_tests_x64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_4.14"
-          - "amzn_5.4"
-          - "amzn_5.10"
-          - "amzn_2023"
+          - "amazon_4.14"
+          - "amazon_5.4"
+          - "amazon_5.10"
+          - "amazon_2023"
           - "fedora_37"
           - "fedora_38"
           - "debian_10"
           - "debian_11"
           - "debian_12"
-          - "centos_79"
+          - "centos_7.9"
           - "centos_8"
         TEST_SET: ["no_tracersuite", "only_tracersuite"]
   after_script:
@@ -261,16 +261,16 @@ kmt_run_sysprobe_tests_arm64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_23.10"
-          - "amzn_4.14"
-          - "amzn_5.4"
-          - "amzn_5.10"
-          - "amzn_2023"
+          - "amazon_4.14"
+          - "amazon_5.4"
+          - "amazon_5.10"
+          - "amazon_2023"
           - "fedora_37"
           - "fedora_38"
           - "debian_10"
           - "debian_11"
           - "debian_12"
-          - "centos_79"
+          - "centos_7.9"
           - "centos_8"
         TEST_SET: ["no_tracersuite", "only_tracersuite"]
   after_script:

tasks/kernel_matrix_testing/download.py

@@ -18,7 +18,7 @@
     requests = None
 
 if TYPE_CHECKING:
-    from tasks.kernel_matrix_testing.types import PathOrStr
+    from tasks.kernel_matrix_testing.types import Arch, PathOrStr
 
 
 def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: str):
@@ -41,23 +41,27 @@ def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: st
     return False
 
 
-def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str):
+def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str, arch: Arch | None = None):
     platforms = get_platforms()
     vmconfig_template = get_vmconfig_template(vmconfig_template_name)
 
     url_base = platforms["url_base"]
 
-    arch = arch_mapping[platform.machine()]
+    if arch is None:
+        arch = arch_mapping[platform.machine()]
     to_download: list[str] = list()
     file_ls: list[str] = list()
     branch_mapping: dict[str, str] = dict()
 
     for tag in platforms[arch]:
-        path = os.path.basename(platforms[arch][tag])
+        platinfo = platforms[arch][tag]
+        if "image" not in platinfo:
+            raise Exit("image is not defined in platform info")
+        path = os.path.basename(platinfo["image"])
         if path.endswith(".xz"):
             path = path[: -len(".xz")]
 
-        branch_mapping[path] = os.path.dirname(platforms[arch][tag]) or "master"
+        branch_mapping[path] = platinfo.get('image_version', 'master')
         file_ls.append(os.path.basename(path))
 
     # if file does not exist download it.
@@ -134,7 +138,11 @@ def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name:
         raise Exit("Failed to set permissions 0766 to rootfs")
 
 
-def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str):
-    download_rootfs(ctx, rootfs_dir, vmconfig_template)
+def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str, all_archs: bool = False):
+    if all_archs:
+        arch_ls: list[Arch] = ["x86_64", "arm64"]
+        for arch in arch_ls:
+            info(f"[+] Updating root filesystem for {arch}")
+            download_rootfs(ctx, rootfs_dir, vmconfig_template, arch)
 
     info("[+] Root filesystem and bootables images updated")

tasks/kernel_matrix_testing/platforms.py

@@ -10,15 +10,19 @@
 from tasks.pipeline import GitlabYamlLoader
 
 if TYPE_CHECKING:
-    from tasks.kernel_matrix_testing.types import Arch, Component, Platforms
+    from tasks.kernel_matrix_testing.types import (
+        Arch,
+        Component,
+        Platforms,
+    )
 
 
 platforms_file = "test/new-e2e/system-probe/config/platforms.json"
 
 
 def get_platforms():
     with open(platforms_file) as f:
-        return cast('Platforms', json.load(f))
+        return cast("Platforms", json.load(f))
 
 
 def filter_by_ci_component(platforms: Platforms, component: Component) -> Platforms:
@@ -33,7 +37,7 @@ def filter_by_ci_component(platforms: Platforms, component: Component) -> Platfo
     new_platforms = platforms.copy()
 
     target_file = (
-        Path(__file__).parent.parent.parent / '.gitlab' / "kernel_matrix_testing" / f"{component.replace('-', '_')}.yml"
+        Path(__file__).parent.parent.parent / ".gitlab" / "kernel_matrix_testing" / f"{component.replace('-', '_')}.yml"
     )
     with open(target_file) as f:
         ci_config = yaml.load(f, Loader=GitlabYamlLoader())
@@ -51,4 +55,8 @@ def filter_by_ci_component(platforms: Platforms, component: Component) -> Platfo
 
         new_platforms[arch] = {k: v for k, v in new_platforms[arch].items() if k in kernels}
 
+        missing_kernels = kernels - set(new_platforms[arch].keys())
+        if missing_kernels:
+            raise Exit(f"Kernels {missing_kernels} not found in {platforms_file} for {arch}")
+
     return new_platforms

tasks/kernel_matrix_testing/types.py

@@ -26,10 +26,20 @@ class DependenciesLayout(TypedDict):  # noqa: F841
     build: Dict[str, DependencyBuild]
 
 
+class PlatformInfo(TypedDict, total=False):
+    os_name: str  # Official OS name  # noqa: F841
+    os_version: str  # Version  # noqa: F841
+    image_version: str  # Image version  # noqa: F841
+    kernel: str  # Kernel version
+    os_id: str  # Short ID for the OS (e.g., "centos" for CentOS)  # noqa: F841
+    image: str  # Name of the image file
+    alt_version_names: List[str]  # Alternative version names (e.g., "jammy" for Ubuntu 22)  # noqa: F841
+
+
 class Platforms(TypedDict):  # noqa: F841
     url_base: str
-    x86_64: Dict[str, str]  # noqa: F841
-    arm64: Dict[str, str]  # noqa: F841
+    x86_64: Dict[str, PlatformInfo]  # noqa: F841
+    arm64: Dict[str, PlatformInfo]  # noqa: F841
 
 
 class Disk(TypedDict):

tasks/kernel_matrix_testing/vars.py

@@ -14,5 +14,6 @@
     "arm": "arm64",
     "aarch64": "arm64",
 }
+arch_ls: list[Arch] = ["x86_64", "arm64"]
 
 VMCONFIG = "vmconfig.json"