diff --git a/tasks/kernel_matrix_testing/vars.py b/tasks/kernel_matrix_testing/vars.py index 10920a98aef38..6eda5fb3bd930 100644 --- a/tasks/kernel_matrix_testing/vars.py +++ b/tasks/kernel_matrix_testing/vars.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Dict, List if TYPE_CHECKING: from tasks.kernel_matrix_testing.types import Arch @@ -14,5 +14,6 @@ "arm": "arm64", "aarch64": "arm64", } +arch_ls: List[Arch] = ["x86_64", "arm64"] VMCONFIG = "vmconfig.json" diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py index c3cabcda60c41..6f5ec4501fdd8 100644 --- a/tasks/kernel_matrix_testing/vmconfig.py +++ b/tasks/kernel_matrix_testing/vmconfig.py @@ -5,7 +5,9 @@ import json import os import platform -from typing import TYPE_CHECKING, Any, List, Optional, Set, Union, cast +import random +from collections import defaultdict +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union, cast from urllib.parse import urlparse from invoke.context import Context @@ -14,7 +16,7 @@ from tasks.kernel_matrix_testing.platforms import filter_by_ci_component, get_platforms from tasks.kernel_matrix_testing.stacks import check_and_get_stack, create_stack, stack_exists from tasks.kernel_matrix_testing.tool import Exit, ask, info, warn -from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_mapping +from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_ls, arch_mapping if TYPE_CHECKING: from tasks.kernel_matrix_testing.types import ( # noqa: F401 @@ -79,45 +81,6 @@ "4.19", "4.20", ] -distributions = { - # Ubuntu mappings - "ubuntu_16": "ubuntu_16.04", - "ubuntu_18": "ubuntu_18.04", - "ubuntu_20": "ubuntu_20.04", - "ubuntu_22": "ubuntu_22.04", - "ubuntu_23": "ubuntu_23.10", - "xenial": "ubuntu_16.04", - "bionic": "ubuntu_18.04", - "focal": "ubuntu_20.04", - "jammy": "ubuntu_22.04", - "mantic": "ubuntu_23.10", - # Amazon Linux mappings - "amazon_4.14": "amzn_4.14", - "amazon_5.4": "amzn_5.4", - "amazon_5.10": "amzn_5.10", - "amzn_4.14": "amzn_4.14", - "amzn_414": "amzn_4.14", - "amzn_5.4": "amzn_5.4", - "amzn_5.10": "amzn_5.10", - "amzn_2023": "amzn_2023", - "amazon_2023": "amzn_2023", - "al3": "amzn_2023", - "amzn_3": "amzn_2023", - # Fedora mappings - "fedora_37": "fedora_37", - "fedora_38": "fedora_38", - # Debian mappings - "debian_10": "debian_10", - "debian_11": "debian_11", - "debian_12": "debian_12", - # CentOS mappings - "centos_79": "centos_79", - "centos_7": "centos_79", - "centos_8": "centos_8", - # Rocky Linux mappings - "rocky_8.5": "rocky_8.5", - "rocky_9.3": "rocky_9.3", -} TICK = "\033[32m\u2713\033[0m" CROSS = "\033[31m\u2718\033[0m" @@ -163,7 +126,7 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]: "x86_64", "arm64", "Alternative names", - "Example VM tag to use with --vms (fuzzy matching)", + "Example VM tags to use with --vms (fuzzy matching)", ] custom_kernels: List[List[str]] = list() for k in kernels: @@ -174,6 +137,7 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]: distro_kernels: List[List[str]] = list() platforms = get_platforms() + mappings = get_distribution_mappings() # Group kernels by name and kernel version, show whether one or two architectures are supported for arch in cast('List[Arch]', ["x86_64", "arm64"]): for name, platinfo in platforms[arch].items(): @@ -187,6 +151,10 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]: entry = row break if entry is None: + names = {k for k, v in mappings.items() if v == name} + # Take two random names for the table so users get an idea of possible mappings + names = random.choices(list(names), k=min(2, len(names))) + entry = [ name, platinfo.get("os_name"), @@ -195,7 +163,7 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]: CROSS, CROSS, ", ".join(platinfo.get("alt_version_names", [])), - f"distro-{name}-{arch}", + ", ".join(f"distro-{n}-{arch}" for n in names), ] distro_kernels.append(entry) @@ -204,6 +172,9 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]: else: entry[5] = TICK + # Sort by name + distro_kernels.sort(key=lambda x: x[0]) + if (not (distro or custom)) or (distro and custom): return [headers] + distro_kernels + custom_kernels elif distro: @@ -230,8 +201,51 @@ def empty_config(file_path: str): f.write(j) +def get_distribution_mappings() -> Dict[str, str]: + platforms = get_platforms() + distro_mappings: Dict[str, str] = dict() + alternative_spellings = {"amzn": ["amazon", "al"]} + mapping_candidates: Dict[str, Set[str]] = defaultdict( + set + ) # Store here maps that could generate duplicates. Values are the possible targets + + for arch in arch_ls: + for name, platinfo in platforms[arch].items(): + if isinstance(platinfo, str): + continue # Avoid a crash if we have the old format in the platforms file + if name in distro_mappings: + continue # Ignore already existing images (from other arch) + + distro_mappings[name] = name # Direct name + distro_mappings[name.replace('.', '')] = name # Allow name without dots + for alt in platinfo.get("alt_version_names", []): + distro_mappings[alt] = name # Alternative version names map directly to the main name + + os_id = platinfo.get("os_id", "") + version = platinfo.get('version', "") + + if version != "": + if ( + os_id != "" and os_id != name.split('_')[0] + ): # If the os_id is different from the main name, add it too + distro_mappings[f"{os_id}_{version}"] = name + + for alt in alternative_spellings.get(os_id, []): + distro_mappings[f"{alt}_{version}"] = name + + name_no_minor_version = f"{os_id}_{version.split('.')[0]}" + mapping_candidates[name_no_minor_version].add(name) + + # Add candidates that didn't have any duplicates + for name, candidates in mapping_candidates.items(): + if len(candidates) == 1: + distro_mappings[name] = candidates.pop() + + return distro_mappings + + def list_possible() -> List[str]: - distros = list(distributions.keys()) + distros = list(get_distribution_mappings().keys()) archs = list(arch_mapping.keys()) archs.append(local_arch) @@ -265,7 +279,7 @@ def normalize_vm_def(possible: List[str], vm: str) -> VMDef: arch = arch_mapping[arch] if recipe == "distro": - version = distributions[version] + version = get_distribution_mappings()[version] elif recipe != "custom": raise Exit(f"Invalid recipe {recipe}")