From b9655b0a5adf942cf8b79bd024d5e07a7a39df79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Thu, 25 Apr 2024 12:14:04 +0200 Subject: [PATCH] [EBPF-474] Improve kernel information in platforms.json (#24097) * Allow downloading images for all architectures * Add update-platform-info task * Improve output of kmt.ls * Adapt to new platforms.json structure * Use guestfs python package * Ignore rescue images when detecting kernel * Ignore empty version names * Fix kernel detection * Use dynamic distribution mappings * Update platform info * Remove empty alternative names * Better help * Fix archs for custom kernels * Fix import order * Fix format * Use manifest files * Improve sorting in kmt.ls * Update platforms.json * Warn when some images have not been updated * Update platforms.json * Update to latest images * Raise an error when image is missing * Fix platforms.json file * Add kmt.validate-platform-info task * review comments * Improve kmt.ls * Show an error when kernels are missing * Fix image names in CI * Switch Amazon 2023 version * Use master version for AL2023 * test with updated names * Update Amazon 2023 name * Add --exclude-matching argument to update-platform-info task * Update platforms.json --- .../kernel_matrix_testing/security_agent.yml | 8 +- .../kernel_matrix_testing/system_probe.yml | 16 +- tasks/kernel_matrix_testing/download.py | 28 +- tasks/kernel_matrix_testing/types.py | 14 +- tasks/kernel_matrix_testing/vars.py | 1 + tasks/kernel_matrix_testing/vmconfig.py | 180 +++++--- tasks/kmt.py | 159 ++++++- .../system-probe/config/platforms.json | 411 ++++++++++++++++-- 8 files changed, 688 insertions(+), 129 deletions(-) diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index 372f00e641290..8f8cb0d0b2b34 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -147,8 +147,8 @@ kmt_run_secagent_tests_x64: - "ubuntu_20.04" - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_10" @@ -181,8 +181,8 @@ kmt_run_secagent_tests_arm64: - TAG: - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_11" diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index 68229f386177c..2f8f570842649 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -225,15 +225,15 @@ kmt_run_sysprobe_tests_x64: - "ubuntu_20.04" - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_4.14" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_4.14" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_10" - "debian_11" - "debian_12" - - "centos_79" + - "centos_7.9" - "centos_8" TEST_SET: ["no_tracersuite", "only_tracersuite"] after_script: @@ -260,15 +260,15 @@ kmt_run_sysprobe_tests_arm64: - "ubuntu_20.04" - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_4.14" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_4.14" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_10" - "debian_11" - "debian_12" - - "centos_79" + - "centos_7.9" - "centos_8" TEST_SET: ["no_tracersuite", "only_tracersuite"] after_script: diff --git a/tasks/kernel_matrix_testing/download.py b/tasks/kernel_matrix_testing/download.py index 0ee38d2002583..9ac1f9d5bfe15 100644 --- a/tasks/kernel_matrix_testing/download.py +++ b/tasks/kernel_matrix_testing/download.py @@ -3,7 +3,7 @@ import os import platform import tempfile -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from invoke.context import Context @@ -18,7 +18,7 @@ requests = None if TYPE_CHECKING: - from tasks.kernel_matrix_testing.types import PathOrStr + from tasks.kernel_matrix_testing.types import Arch, PathOrStr def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: str): @@ -41,23 +41,27 @@ def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: st return False -def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str): +def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str, arch: Arch | None = None): platforms = get_platforms() vmconfig_template = get_vmconfig_template(vmconfig_template_name) url_base = platforms["url_base"] - arch = arch_mapping[platform.machine()] - to_download: List[str] = list() - file_ls: List[str] = list() + if arch is None: + arch = arch_mapping[platform.machine()] + to_download: list[str] = list() + file_ls: list[str] = list() branch_mapping: dict[str, str] = dict() for tag in platforms[arch]: - path = os.path.basename(platforms[arch][tag]) + platinfo = platforms[arch][tag] + if "image" not in platinfo: + raise Exit("image is not defined in platform info") + path = os.path.basename(platinfo["image"]) if path.endswith(".xz"): path = path[: -len(".xz")] - branch_mapping[path] = os.path.dirname(platforms[arch][tag]) or "master" + branch_mapping[path] = platinfo.get('image_version', 'master') file_ls.append(os.path.basename(path)) # if file does not exist download it. @@ -134,7 +138,11 @@ def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: raise Exit("Failed to set permissions 0766 to rootfs") -def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str): - download_rootfs(ctx, rootfs_dir, vmconfig_template) +def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str, all_archs: bool = False): + if all_archs: + arch_ls: list[Arch] = ["x86_64", "arm64"] + for arch in arch_ls: + info(f"[+] Updating root filesystem for {arch}") + download_rootfs(ctx, rootfs_dir, vmconfig_template, arch) info("[+] Root filesystem and bootables images updated") diff --git a/tasks/kernel_matrix_testing/types.py b/tasks/kernel_matrix_testing/types.py index af42a6d695810..6f25f425823a9 100644 --- a/tasks/kernel_matrix_testing/types.py +++ b/tasks/kernel_matrix_testing/types.py @@ -25,10 +25,20 @@ class DependenciesLayout(TypedDict): # noqa: F841 build: Dict[str, DependencyBuild] +class PlatformInfo(TypedDict, total=False): + os_name: str # Official OS name # noqa: F841 + os_version: str # Version # noqa: F841 + image_version: str # Image version # noqa: F841 + kernel: str # Kernel version + os_id: str # Short ID for the OS (e.g., "centos" for CentOS) # noqa: F841 + image: str # Name of the image file + alt_version_names: List[str] # Alternative version names (e.g., "jammy" for Ubuntu 22) # noqa: F841 + + class Platforms(TypedDict): # noqa: F841 url_base: str - x86_64: Dict[str, str] # noqa: F841 - arm64: Dict[str, str] # noqa: F841 + x86_64: Dict[str, PlatformInfo] # noqa: F841 + arm64: Dict[str, PlatformInfo] # noqa: F841 class Disk(TypedDict): diff --git a/tasks/kernel_matrix_testing/vars.py b/tasks/kernel_matrix_testing/vars.py index 10920a98aef38..222464fa827e8 100644 --- a/tasks/kernel_matrix_testing/vars.py +++ b/tasks/kernel_matrix_testing/vars.py @@ -14,5 +14,6 @@ "arm": "arm64", "aarch64": "arm64", } +arch_ls: list[Arch] = ["x86_64", "arm64"] VMCONFIG = "vmconfig.json" diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py index 75c90b7ae3dcc..711586f7bc983 100644 --- a/tasks/kernel_matrix_testing/vmconfig.py +++ b/tasks/kernel_matrix_testing/vmconfig.py @@ -5,6 +5,8 @@ import json import os import platform +import random +from collections import defaultdict from typing import TYPE_CHECKING, Any, List, Optional, Set, Union, cast from urllib.parse import urlparse @@ -14,7 +16,7 @@ from tasks.kernel_matrix_testing.platforms import get_platforms from tasks.kernel_matrix_testing.stacks import check_and_get_stack, create_stack, stack_exists from tasks.kernel_matrix_testing.tool import Exit, ask, info, warn -from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_mapping +from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_ls, arch_mapping if TYPE_CHECKING: from tasks.kernel_matrix_testing.types import ( # noqa: F401 @@ -78,48 +80,9 @@ "4.19", "4.20", ] -distributions = { - # Ubuntu mappings - "ubuntu_16": "ubuntu_16.04", - "ubuntu_18": "ubuntu_18.04", - "ubuntu_20": "ubuntu_20.04", - "ubuntu_22": "ubuntu_22.04", - "ubuntu_23": "ubuntu_23.10", - "xenial": "ubuntu_16.04", - "bionic": "ubuntu_18.04", - "focal": "ubuntu_20.04", - "jammy": "ubuntu_22.04", - "mantic": "ubuntu_23.10", - # Amazon Linux mappings - "amazon_4.14": "amzn_4.14", - "amazon_5.4": "amzn_5.4", - "amazon_5.10": "amzn_5.10", - "amzn_4.14": "amzn_4.14", - "amzn_414": "amzn_4.14", - "amzn_5.4": "amzn_5.4", - "amzn_5.10": "amzn_5.10", - "amzn_2023": "amzn_2023", - "amazon_2023": "amzn_2023", - "al3": "amzn_2023", - "amzn_3": "amzn_2023", - # Fedora mappings - "fedora_37": "fedora_37", - "fedora_38": "fedora_38", - # Debian mappings - "debian_10": "debian_10", - "debian_11": "debian_11", - "debian_12": "debian_12", - # CentOS mappings - "centos_79": "centos_79", - "centos_7": "centos_79", - "centos_8": "centos_8", - # Rocky Linux mappings - "rocky_8.5": "rocky_8.5", - "rocky_9.3": "rocky_9.3", -} - -TICK = "\u2713" -CROSS = "\u2718" + +TICK = "\033[32m\u2713\033[0m" +CROSS = "\033[31m\u2718\033[0m" table = [ ["Image", "x86_64", "arm64"], ["ubuntu-18 (bionic)", TICK, CROSS], @@ -153,22 +116,71 @@ def lte_414(version: str) -> bool: return (int(major) <= 4) and (int(minor) <= 14) -def get_image_list(distro: bool, custom: bool) -> List[List[str]]: - custom_kernels: List[List[str]] = list() - for k in kernels: +def get_image_list(distro: bool, custom: bool) -> list[list[str]]: + headers = [ + "VM name", + "OS Name", + "OS Version", + "Kernel", + "x86_64", + "arm64", + "Alternative names", + "Example VM tags to use with --vms (fuzzy matching)", + ] + custom_kernels: list[list[str]] = list() + for k in sorted(kernels, key=lambda x: tuple(map(int, x.split('.')))): if lte_414(k): - custom_kernels.append([f"custom kernel v{k}", TICK, CROSS]) + custom_kernels.append([f"custom-{k}", "Debian", "Custom", k, TICK, CROSS, "", f"custom-{k}-x86_64"]) else: - custom_kernels.append([f"custom kernel v{k}", TICK, TICK]) - - if (not (distro or custom)) or (distro and custom): - return table + custom_kernels - elif distro: - return table - elif custom: - return custom_kernels - else: - return [] + custom_kernels.append([f"custom-{k}", "Debian", "Custom", k, TICK, TICK, "", f"custom-{k}-x86_64"]) + + distro_kernels: list[list[str]] = list() + platforms = get_platforms() + mappings = get_distribution_mappings() + # Group kernels by name and kernel version, show whether one or two architectures are supported + for arch in arch_ls: + for name, platinfo in platforms[arch].items(): + if isinstance(platinfo, str): + continue # Old format + + # See if we've already added this kernel but for a different architecture. If not, create the entry. + entry = None + for row in distro_kernels: + if row[0] == name and row[3] == platinfo.get('kernel'): + entry = row + break + if entry is None: + names = {k for k, v in mappings.items() if v == name} + # Take two random names for the table so users get an idea of possible mappings + names = random.choices(list(names), k=min(2, len(names))) + + entry = [ + name, + platinfo.get("os_name"), + platinfo.get("os_version"), + platinfo.get("kernel"), + CROSS, + CROSS, + ", ".join(platinfo.get("alt_version_names", [])), + ", ".join(f"distro-{n}-{arch}" for n in names), + ] + distro_kernels.append(entry) + + if arch == "x86_64": + entry[4] = TICK + else: + entry[5] = TICK + + # Sort by name + distro_kernels.sort(key=lambda x: x[0]) + + table = [headers] + if distro: + table += distro_kernels + if custom: + table += custom_kernels + + return table def check_memory_and_vcpus(memory: List[Any], vcpus: List[Any]): @@ -187,8 +199,51 @@ def empty_config(file_path: str): f.write(j) -def list_possible() -> List[str]: - distros = list(distributions.keys()) +def get_distribution_mappings() -> dict[str, str]: + platforms = get_platforms() + distro_mappings: dict[str, str] = dict() + alternative_spellings = {"amzn": ["amazon", "al"]} + mapping_candidates: dict[str, set[str]] = defaultdict( + set + ) # Store here maps that could generate duplicates. Values are the possible targets + + for arch in arch_ls: + for name, platinfo in platforms[arch].items(): + if isinstance(platinfo, str): + continue # Avoid a crash if we have the old format in the platforms file + if name in distro_mappings: + continue # Ignore already existing images (from other arch) + + distro_mappings[name] = name # Direct name + distro_mappings[name.replace('.', '')] = name # Allow name without dots + for alt in platinfo.get("alt_version_names", []): + distro_mappings[alt] = name # Alternative version names map directly to the main name + + os_id = platinfo.get("os_id", "") + version = platinfo.get('version', "") + + if version != "": + if ( + os_id != "" and os_id != name.split('_')[0] + ): # If the os_id is different from the main name, add it too + distro_mappings[f"{os_id}_{version}"] = name + + for alt in alternative_spellings.get(os_id, []): + distro_mappings[f"{alt}_{version}"] = name + + name_no_minor_version = f"{os_id}_{version.split('.')[0]}" + mapping_candidates[name_no_minor_version].add(name) + + # Add candidates that didn't have any duplicates + for name, candidates in mapping_candidates.items(): + if len(candidates) == 1: + distro_mappings[name] = candidates.pop() + + return distro_mappings + + +def list_possible() -> list[str]: + distros = list(get_distribution_mappings().keys()) archs = list(arch_mapping.keys()) archs.append(local_arch) @@ -222,7 +277,7 @@ def normalize_vm_def(possible: List[str], vm: str) -> VMDef: arch = arch_mapping[arch] if recipe == "distro": - version = distributions[version] + version = get_distribution_mappings()[version] elif recipe != "custom": raise Exit(f"Invalid recipe {recipe}") @@ -273,7 +328,10 @@ def get_kernel_config( arch = arch_mapping[platform.machine()] url_base = platforms["url_base"] - kernel_path = platforms[arch][version] + platinfo = platforms[arch][version] + if "image" not in platinfo or "image_version" not in platinfo: + raise Exit(f"image not found in platform information for {version}") + kernel_path = f"{platinfo['image_version']}/{platinfo['image']}" kernel_name = xz_suffix_removed(os.path.basename(kernel_path)) return {"tag": version, "image_source": os.path.join(url_base, kernel_path), "dir": kernel_name} diff --git a/tasks/kmt.py b/tasks/kmt.py index 42d267a0f7064..b5705e3274e10 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -22,8 +22,10 @@ from tasks.kernel_matrix_testing.infra import HostInstance, LibvirtDomain, build_infrastructure from tasks.kernel_matrix_testing.init_kmt import init_kernel_matrix_testing_system from tasks.kernel_matrix_testing.kmt_os import get_kmt_os +from tasks.kernel_matrix_testing.platforms import get_platforms, platforms_file from tasks.kernel_matrix_testing.stacks import check_and_get_stack, ec2_instance_ids from tasks.kernel_matrix_testing.tool import Exit, ask, info, warn +from tasks.kernel_matrix_testing.vars import arch_ls from tasks.libs.common.gitlab import Gitlab, get_gitlab_token from tasks.system_probe import EMBEDDED_SHARE_DIR @@ -247,7 +249,7 @@ def init(ctx: Context, lite=False): @task -def update_resources(ctx: Context, vmconfig_template="system-probe"): +def update_resources(ctx: Context, vmconfig_template="system-probe", all_archs=False): kmt_os = get_kmt_os() warn("Updating resource dependencies will delete all running stacks.") @@ -257,7 +259,7 @@ def update_resources(ctx: Context, vmconfig_template="system-probe"): for stack in glob(f"{kmt_os.stacks_dir}/*"): destroy_stack(ctx, stack=os.path.basename(stack)) - update_rootfs(ctx, kmt_os.rootfs_dir, vmconfig_template) + update_rootfs(ctx, kmt_os.rootfs_dir, vmconfig_template, all_archs=all_archs) @task @@ -771,3 +773,156 @@ def status(ctx: Context, stack: Optional[str] = None, all=False): for line in lines: print(line) print("") + + +@task( + help={ + "version": "The version to update the images to. If not provided, version will not be changed. If 'latest' is provided, the latest version will be used.", + "update-only-matching": "Only update the platform info for images that match the given regex", + "exclude-matching": "Exclude images that match the given regex", + } +) +def update_platform_info( + ctx: Context, + version: str | None = None, + update_only_matching: str | None = None, + exclude_matching: str | None = None, +): + """Generate a JSON file with platform information for all the images + found in the KMT S3 bucket. + """ + res = ctx.run( + "aws-vault exec sso-staging-engineering -- aws s3 ls --recursive s3://dd-agent-omnibus/kernel-version-testing/rootfs", + warn=True, + ) + if res is None or not res.ok: + raise Exit("Cannot list bucket contents") + + objects = [line.split()[-1] for line in res.stdout.splitlines()] + objects_by_version: dict[str, list[str]] = defaultdict(list) + + for obj in objects: + v = "/".join(obj.split("/")[2:-1]) + if v != "": + objects_by_version[v].append(obj) + + if version is None: + master_versions = [v for v in objects_by_version if re.match(r"^20[0-9]{6}_[0-9a-f]+$", v)] + if len(master_versions) == 0: + raise Exit("No master versions available") + + version = sorted(master_versions)[-1] + info(f"[+] detected {version} as latest version from master branch") + + if version not in objects_by_version: + raise Exit(f"Version {version} not found in S3 bucket, cannot update") + + manifests = [obj for obj in objects_by_version[version] if obj.endswith(".manifest")] + platforms = get_platforms() + + with tempfile.TemporaryDirectory() as tmpdir: + for manifest in manifests: + info(f"[+] Processing manifest {manifest}") + ctx.run(f"aws-vault exec sso-staging-engineering -- aws s3 cp s3://dd-agent-omnibus/{manifest} {tmpdir}") + with open(f"{tmpdir}/{os.path.basename(manifest)}") as f: + options = f.readlines() + keyvals = {line.split("=")[0]: line.split("=")[1].strip().strip('"') for line in options} + + try: + arch = arch_mapping[keyvals['ARCH']] + image_name = keyvals['IMAGE_NAME'] + image_filename = keyvals['IMAGE_FILENAME'] + except KeyError: + raise Exit(f"[!] Invalid manifest {manifest}") + + if arch not in platforms: + warn(f"[!] Unsupported architecture {arch}, skipping") + continue + + if update_only_matching is not None and re.search(update_only_matching, image_name) is None: + warn(f"[!] Image {image_name} does not match the filter, skipping") + continue + + if exclude_matching is not None and re.search(exclude_matching, image_name) is not None: + warn(f"[!] Image {image_name} matches the exclude filter, skipping") + continue + + manifest_to_platinfo_keys = { + 'NAME': 'os_name', + 'ID': 'os_id', + 'KERNEL_VERSION': 'kernel', + 'VERSION_ID': 'os_version', + } + + if image_name not in platforms[arch]: + platforms[arch][image_name] = {} + + for mkey, pkey in manifest_to_platinfo_keys.items(): + if mkey in keyvals: + platforms[arch][image_name][pkey] = keyvals[mkey] + + platforms[arch][image_name]['image'] = image_filename + ".xz" + platforms[arch][image_name]['image_version'] = version + + if 'VERSION_CODENAME' in keyvals: + altname = keyvals['VERSION_CODENAME'] + # Do not modify existing altnames + altnames = platforms[arch][image_name].get('alt_version_names', []) + if altname not in altnames: + altnames.append(altname) + + platforms[arch][image_name]['alt_version_names'] = altnames + + info(f"[+] Writing output to {platforms_file}...") + + # Do validation of the platforms dict, check that there are no outdated versions + for arch in arch_ls: + for image_name, platinfo in platforms[arch].items(): + if update_only_matching is not None and re.search(update_only_matching, image_name) is None: + continue # Only validate those that match + + if exclude_matching is not None and re.search(exclude_matching, image_name) is not None: + continue + + version_from_file = platinfo.get('image_version') + if version_from_file != version: + warn( + f"[!] Image {image_name} ({arch}) has version {version_from_file} but we are updating to {version}, manifest file may be missing?" + ) + + with open(platforms_file, "w") as f: + json.dump(platforms, f, indent=2) + + +@task +def validate_platform_info(ctx: Context): + """Validate the platform info file for correctness, ensuring that all images can be found""" + platforms = get_platforms() + errors: set[str] = set() + + for arch in arch_ls: + for image_name, platinfo in platforms[arch].items(): + image = platinfo.get('image') + if image is None: + warn(f"[!] {image_name} does not have an image filename") + errors.add(image_name) + continue + + version = platinfo.get('image_version') + if version is None: + warn(f"[!] {image_name} does not have an image version") + errors.add(image_name) + continue + + remote = f"{platforms['url_base']}/{version}/{image}" + res = ctx.run(f"curl -s -I {remote}") + if res is None or not res.ok: + warn(f"[!] {image_name}: {image} for version {version} not found at {remote}") + errors.add(image_name) + else: + info(f"[+] {image_name}: {image} for version {version} found at {remote}") + + if len(errors) == 0: + info("[+] Platform info file is valid") + else: + raise Exit(f"[!] Found {len(errors)} errors in the platform info file. Images failed: {', '.join(errors)}") diff --git a/test/new-e2e/system-probe/config/platforms.json b/test/new-e2e/system-probe/config/platforms.json index d6069e0a88e80..84dc0ba917ae4 100644 --- a/test/new-e2e/system-probe/config/platforms.json +++ b/test/new-e2e/system-probe/config/platforms.json @@ -1,45 +1,372 @@ { - "url_base": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs", - "x86_64": { - "ubuntu_16.04": "master/ubuntu-16-x86_64.qcow2.xz", - "ubuntu_18.04": "master/ubuntu-18-x86_64.qcow2.xz", - "ubuntu_20.04": "master/ubuntu-20-x86_64.qcow2.xz", - "ubuntu_22.04": "master/ubuntu-22-x86_64.qcow2.xz", - "ubuntu_23.10": "master/ubuntu-23-x86_64.qcow2.xz", - "amzn_4.14": "master/amzn2-kvm-2.0-x86_64-4.14.qcow2.xz", - "amzn_5.4": "master/amzn2-kvm-2.0-x86_64-5.4.qcow2.xz", - "amzn_5.10": "master/amzn2-kvm-2.0-x86_64-5.10.qcow2.xz", - "fedora_37": "master/Fedora-Cloud-Base-37.x86_64.qcow2.xz", - "fedora_38": "master/Fedora-Cloud-Base-38.x86_64.qcow2.xz", - "debian_10": "master/debian-10-generic-x86_64.qcow2.xz", - "debian_11": "master/debian-11-generic-x86_64.qcow2.xz", - "debian_12": "master/debian-12-generic-x86_64.qcow2.xz", - "centos_79": "master/centos-7.9-x86_64.qcow2.xz", - "centos_8": "master/centos-8-x86_64.qcow2.xz", - "oracle_7.9": "master/oracle-7.9-x86_64.qcow.xz", - "oracle_8.9": "master/oracle-8.9-x86_64.qcow.xz", - "oracle_9.3": "master/oracle-9.3-x86_64.qcow.xz", - "rocky_8.5": "master/rocky-8.5-x86_64.qcow2.xz", - "rocky_9.3": "master/rocky-9.3-x86_64.qcow2.xz" - }, - "arm64": { - "ubuntu_18.04": "master/ubuntu-18-arm64.qcow2.xz", - "ubuntu_20.04": "master/ubuntu-20-arm64.qcow2.xz", - "ubuntu_22.04": "master/ubuntu-22-arm64.qcow2.xz", - "ubuntu_23.10": "master/ubuntu-23-arm64.qcow2.xz", - "amzn_4.14": "master/amzn2-kvm-2.0-arm64-4.14.qcow2.xz", - "amzn_5.4": "master/amzn2-kvm-2.0-arm64-5.4.qcow2.xz", - "amzn_5.10": "master/amzn2-kvm-2.0-arm64-5.10.qcow2.xz", - "fedora_37": "master/Fedora-Cloud-Base-37.arm64.qcow2.xz", - "fedora_38": "master/Fedora-Cloud-Base-38.arm64.qcow2.xz", - "debian_10": "master/debian-10-generic-arm64.qcow2.xz", - "debian_11": "master/debian-11-generic-arm64.qcow2.xz", - "debian_12": "master/debian-12-generic-arm64.qcow2.xz", - "centos_79": "master/centos-7.9-arm64.qcow2.xz", - "centos_8": "master/centos-8-arm64.qcow2.xz", - "oracle_8.9": "master/oracle-8.9-arm64.qcow.xz", - "oracle_9.3": "master/oracle-9.3-arm64.qcow.xz", - "rocky_8.5": "master/rocky-8.5-arm64.qcow2.xz", - "rocky_9.3": "master/rocky-9.3-arm64.qcow2.xz" + "url_base": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs", + "x86_64": { + "fedora_37": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.0.7-301.fc37", + "os_version": "37", + "image": "Fedora-Cloud-Base-37.x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "fedora_38": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.2.9-300.fc38", + "os_version": "38", + "image": "Fedora-Cloud-Base-38.x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_4.14": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "4.14.314-237.533.amzn2", + "os_version": "2", + "image": "amzn2-x86_64-4.14.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.10": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.10.214-202.855.amzn2", + "os_version": "2", + "image": "amzn2-x86_64-5.10.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.4": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.4.273-186.370.amzn2", + "os_version": "2", + "image": "amzn2-x86_64-5.4.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_2023": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "6.1.77-99.164.amzn2023", + "os_version": "2023", + "image": "amzn2023-amd64-2023.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_7.9": { + "os_name": "CentOS Linux", + "os_id": "centos", + "kernel": "3.10.0-1160.80.1.el7", + "os_version": "7.9.2009", + "image": "centos-7.9-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_8": { + "os_name": "CentOS Stream", + "os_id": "centos", + "kernel": "4.18.0-545.el8", + "os_version": "8", + "image": "centos-8-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "debian_10": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "4.19.0-26", + "os_version": "10", + "image": "debian-10-generic-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "buster" + ] + }, + "debian_11": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "5.10.0-28", + "os_version": "11", + "image": "debian-11-generic-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bullseye" + ] + }, + "debian_12": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "6.1.0-18", + "os_version": "12", + "image": "debian-12-generic-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bookworm" + ] + }, + "oracle_7.9": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "3.10.0-1160.105.1.0.1.el7", + "os_version": "7.9", + "image": "oracle-7.9-x86_64.qcow.xz", + "image_version": "20240424_95349298" + }, + "oracle_8.9": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el8uek", + "os_version": "8.9", + "image": "oracle-8.9-x86_64.qcow.xz", + "image_version": "20240424_95349298" + }, + "oracle_9.3": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el9uek", + "os_version": "9.3", + "image": "oracle-9.3-x86_64.qcow.xz", + "image_version": "20240424_95349298" + }, + "rocky_8.5": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "4.18.0-348.el8.0.2", + "os_version": "8.5", + "image": "rocky-8.5-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "rocky_9.3": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "5.14.0-362.8.1.el9_3", + "os_version": "9.3", + "image": "rocky-9.3-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "ubuntu_16.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "4.4.0-210-generic", + "os_version": "16.04", + "image": "ubuntu-16.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "xenial" + ] + }, + "ubuntu_18.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "4.18.0-25-generic", + "os_version": "18.04", + "image": "ubuntu-18.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bionic" + ] + }, + "ubuntu_20.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.4.0-167-generic", + "os_version": "20.04", + "image": "ubuntu-20.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "focal" + ] + }, + "ubuntu_22.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.15.0-91-generic", + "os_version": "22.04", + "image": "ubuntu-22.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "jammy" + ] + }, + "ubuntu_23.10": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "6.5.0-9-generic", + "os_version": "23.10", + "image": "ubuntu-23.10-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "mantic" + ] + } + }, + "arm64": { + "fedora_37": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.0.7-301.fc37", + "os_version": "37", + "image": "Fedora-Cloud-Base-37.arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "fedora_38": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.2.9-300.fc38", + "os_version": "38", + "image": "Fedora-Cloud-Base-38.arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_4.14": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "4.14.314-237.533.amzn2", + "os_version": "2", + "image": "amzn2-arm64-4.14.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.4": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.4.273-186.370.amzn2", + "os_version": "2", + "image": "amzn2-arm64-5.4.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_7.9": { + "os_name": "CentOS Linux", + "os_id": "centos", + "kernel": "4.18.0-348.20.1.el7", + "os_version": "7.9.2009", + "image": "centos-7.9-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_8": { + "os_name": "CentOS Stream", + "os_id": "centos", + "kernel": "4.18.0-545.el8", + "os_version": "8", + "image": "centos-8-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "debian_10": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "4.19.0-26", + "os_version": "10", + "image": "debian-10-generic-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "buster" + ] + }, + "debian_11": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "5.10.0-28", + "os_version": "11", + "image": "debian-11-generic-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bullseye" + ] + }, + "debian_12": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "6.1.0-18", + "os_version": "12", + "image": "debian-12-generic-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bookworm" + ] + }, + "oracle_8.9": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el8uek", + "os_version": "8.9", + "image": "oracle-8.9-arm64.qcow.xz", + "image_version": "20240424_95349298" + }, + "oracle_9.3": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el9uek", + "os_version": "9.3", + "image": "oracle-9.3-arm64.qcow.xz", + "image_version": "20240424_95349298" + }, + "rocky_8.5": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "4.18.0-348.el8.0.2", + "os_version": "8.5", + "image": "rocky-8.5-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "rocky_9.3": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "5.14.0-362.8.1.el9_3", + "os_version": "9.3", + "image": "rocky-9.3-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.10": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.10.214-202.855.amzn2", + "os_version": "2", + "image": "amzn2-arm64-5.10.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_2023": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "6.1.77-99.164.amzn2023", + "os_version": "2023", + "image": "amzn2023-arm64-2023.qcow2.xz", + "image_version": "20240424_95349298" + }, + "ubuntu_18.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "4.18.0-25-generic", + "os_version": "18.04", + "image": "ubuntu-18.04-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bionic" + ] + }, + "ubuntu_20.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.4.0-167-generic", + "os_version": "20.04", + "image": "ubuntu-20.04-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "focal" + ] + }, + "ubuntu_22.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.15.0-91-generic", + "os_version": "22.04", + "image": "ubuntu-22.04-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "jammy" + ] + }, + "ubuntu_23.10": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "6.5.0-9-generic", + "os_version": "23.10", + "image": "ubuntu-23.10-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "mantic" + ] } + } }