Use dynamic distribution mappings

tasks/kernel_matrix_testing/vars.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, List
 
 if TYPE_CHECKING:
     from tasks.kernel_matrix_testing.types import Arch
@@ -14,5 +14,6 @@
     "arm": "arm64",
     "aarch64": "arm64",
 }
+arch_ls: List[Arch] = ["x86_64", "arm64"]
 
 VMCONFIG = "vmconfig.json"

tasks/kernel_matrix_testing/vmconfig.py

@@ -5,7 +5,9 @@
 import json
 import os
 import platform
-from typing import TYPE_CHECKING, Any, List, Optional, Set, Union, cast
+import random
+from collections import defaultdict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union, cast
 from urllib.parse import urlparse
 
 from invoke.context import Context
@@ -14,7 +16,7 @@
 from tasks.kernel_matrix_testing.platforms import filter_by_ci_component, get_platforms
 from tasks.kernel_matrix_testing.stacks import check_and_get_stack, create_stack, stack_exists
 from tasks.kernel_matrix_testing.tool import Exit, ask, info, warn
-from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_mapping
+from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_ls, arch_mapping
 
 if TYPE_CHECKING:
     from tasks.kernel_matrix_testing.types import (  # noqa: F401
@@ -79,45 +81,6 @@
     "4.19",
     "4.20",
 ]
-distributions = {
-    # Ubuntu mappings
-    "ubuntu_16": "ubuntu_16.04",
-    "ubuntu_18": "ubuntu_18.04",
-    "ubuntu_20": "ubuntu_20.04",
-    "ubuntu_22": "ubuntu_22.04",
-    "ubuntu_23": "ubuntu_23.10",
-    "xenial": "ubuntu_16.04",
-    "bionic": "ubuntu_18.04",
-    "focal": "ubuntu_20.04",
-    "jammy": "ubuntu_22.04",
-    "mantic": "ubuntu_23.10",
-    # Amazon Linux mappings
-    "amazon_4.14": "amzn_4.14",
-    "amazon_5.4": "amzn_5.4",
-    "amazon_5.10": "amzn_5.10",
-    "amzn_4.14": "amzn_4.14",
-    "amzn_414": "amzn_4.14",
-    "amzn_5.4": "amzn_5.4",
-    "amzn_5.10": "amzn_5.10",
-    "amzn_2023": "amzn_2023",
-    "amazon_2023": "amzn_2023",
-    "al3": "amzn_2023",
-    "amzn_3": "amzn_2023",
-    # Fedora mappings
-    "fedora_37": "fedora_37",
-    "fedora_38": "fedora_38",
-    # Debian mappings
-    "debian_10": "debian_10",
-    "debian_11": "debian_11",
-    "debian_12": "debian_12",
-    # CentOS mappings
-    "centos_79": "centos_79",
-    "centos_7": "centos_79",
-    "centos_8": "centos_8",
-    # Rocky Linux mappings
-    "rocky_8.5": "rocky_8.5",
-    "rocky_9.3": "rocky_9.3",
-}
 
 TICK = "\033[32m\u2713\033[0m"
 CROSS = "\033[31m\u2718\033[0m"
@@ -163,7 +126,7 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]:
         "x86_64",
         "arm64",
         "Alternative names",
-        "Example VM tag to use with --vms (fuzzy matching)",
+        "Example VM tags to use with --vms (fuzzy matching)",
     ]
     custom_kernels: List[List[str]] = list()
     for k in kernels:
@@ -174,6 +137,7 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]:
 
     distro_kernels: List[List[str]] = list()
     platforms = get_platforms()
+    mappings = get_distribution_mappings()
     # Group kernels by name and kernel version, show whether one or two architectures are supported
     for arch in cast('List[Arch]', ["x86_64", "arm64"]):
         for name, platinfo in platforms[arch].items():
@@ -187,6 +151,10 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]:
                     entry = row
                     break
             if entry is None:
+                names = {k for k, v in mappings.items() if v == name}
+                # Take two random names for the table so users get an idea of possible mappings
+                names = random.choices(list(names), k=min(2, len(names)))
+
                 entry = [
                     name,
                     platinfo.get("os_name"),
@@ -195,7 +163,7 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]:
                     CROSS,
                     CROSS,
                     ", ".join(platinfo.get("alt_version_names", [])),
-                    f"distro-{name}-{arch}",
+                    ", ".join(f"distro-{n}-{arch}" for n in names),
                 ]
                 distro_kernels.append(entry)
 
@@ -204,6 +172,9 @@ def get_image_list(distro: bool, custom: bool) -> List[List[str]]:
             else:
                 entry[5] = TICK
 
+    # Sort by name
+    distro_kernels.sort(key=lambda x: x[0])
+
     if (not (distro or custom)) or (distro and custom):
         return [headers] + distro_kernels + custom_kernels
     elif distro:
@@ -230,8 +201,51 @@ def empty_config(file_path: str):
         f.write(j)
 
 
+def get_distribution_mappings() -> Dict[str, str]:
+    platforms = get_platforms()
+    distro_mappings: Dict[str, str] = dict()
+    alternative_spellings = {"amzn": ["amazon", "al"]}
+    mapping_candidates: Dict[str, Set[str]] = defaultdict(
+        set
+    )  # Store here maps that could generate duplicates. Values are the possible targets
+
+    for arch in arch_ls:
+        for name, platinfo in platforms[arch].items():
+            if isinstance(platinfo, str):
+                continue  # Avoid a crash if we have the old format in the platforms file
+            if name in distro_mappings:
+                continue  # Ignore already existing images (from other arch)
+
+            distro_mappings[name] = name  # Direct name
+            distro_mappings[name.replace('.', '')] = name  # Allow name without dots
+            for alt in platinfo.get("alt_version_names", []):
+                distro_mappings[alt] = name  # Alternative version names map directly to the main name
+
+            os_id = platinfo.get("os_id", "")
+            version = platinfo.get('version', "")
+
+            if version != "":
+                if (
+                    os_id != "" and os_id != name.split('_')[0]
+                ):  # If the os_id is different from the main name, add it too
+                    distro_mappings[f"{os_id}_{version}"] = name
+
+                for alt in alternative_spellings.get(os_id, []):
+                    distro_mappings[f"{alt}_{version}"] = name
+
+                name_no_minor_version = f"{os_id}_{version.split('.')[0]}"
+                mapping_candidates[name_no_minor_version].add(name)
+
+    # Add candidates that didn't have any duplicates
+    for name, candidates in mapping_candidates.items():
+        if len(candidates) == 1:
+            distro_mappings[name] = candidates.pop()
+
+    return distro_mappings
+
+
 def list_possible() -> List[str]:
-    distros = list(distributions.keys())
+    distros = list(get_distribution_mappings().keys())
     archs = list(arch_mapping.keys())
     archs.append(local_arch)
 
@@ -265,7 +279,7 @@ def normalize_vm_def(possible: List[str], vm: str) -> VMDef:
         arch = arch_mapping[arch]
 
     if recipe == "distro":
-        version = distributions[version]
+        version = get_distribution_mappings()[version]
     elif recipe != "custom":
         raise Exit(f"Invalid recipe {recipe}")