Include python version in test matrix for cross version tests (mlflow…

…#11433)

Signed-off-by: harupy <[email protected]>

.github/workflows/cross-version-tests.yml

@@ -133,14 +133,9 @@ jobs:
           ref: ${{ github.event.inputs.ref }}
       - uses: ./.github/actions/free-disk-space
         if: matrix.free_disk_space
-      - name: Get python version
-        id: get-python-version
-        run: |
-          python_version=$(python dev/get_minimum_required_python.py -p ${{ matrix.package }} -v ${{ matrix.version }} --python-versions "3.8,3.9")
-          echo "version=$python_version" >> $GITHUB_OUTPUT
       - uses: ./.github/actions/setup-python
         with:
-          python-version: ${{ steps.get-python-version.outputs.version }}
+          python-version: ${{ matrix.python }}
       - uses: ./.github/actions/setup-pyenv
       - uses: ./.github/actions/setup-java
         with:

dev/set_matrix.py

@@ -70,6 +70,7 @@ class TestConfig(BaseModel):
     maximum: Version
     unsupported: t.Optional[t.List[Version]] = None
     requirements: t.Optional[t.Dict[str, t.List[str]]] = None
+    python: t.Optional[t.Dict[str, str]] = None
     java: t.Optional[t.Dict[str, str]] = None
     run: str
     allow_unreleased_max_version: t.Optional[bool] = None
@@ -100,6 +101,7 @@ class MatrixItem(BaseModel):
     run: str
     package: str
     version: Version
+    python: str
     java: str
     supported: bool
     free_disk_space: bool
@@ -128,12 +130,8 @@ def read_yaml(location, if_error=None):
         raise
 
 
-@functools.lru_cache
 def get_released_versions(package_name):
-    url = f"https://pypi.org/pypi/{package_name}/json"
-    response = requests.get(url)
-    response.raise_for_status()
-    data = response.json()
+    data = pypi_json(package_name)
     versions = []
     for version, distributions in data["releases"].items():
         if len(distributions) == 0 or any(d.get("yanked", False) for d in distributions):
@@ -246,6 +244,41 @@ def get_java_version(java: t.Optional[t.Dict[str, str]], version: str) -> str:
     return default
 
 
+@functools.lru_cache(maxsize=128)
+def pypi_json(package: str) -> t.Dict[str, t.Any]:
+    resp = requests.get(f"https://pypi.org/pypi/{package}/json")
+    resp.raise_for_status()
+    return resp.json()
+
+
+def get_requires_python(package: str, version: str) -> str:
+    package_json = pypi_json(package)
+    requires_python = next(
+        (
+            distributions[0].get("requires_python")
+            for ver, distributions in package_json["releases"].items()
+            if ver == version and distributions
+        ),
+        None,
+    )
+    candidates = ("3.8", "3.9")
+    if requires_python is None:
+        return candidates[0]
+
+    spec = SpecifierSet(requires_python)
+    return next((c for c in candidates if spec.contains(c)), None) or candidates[0]
+
+
+def get_python_version(python: t.Optional[t.Dict[str, str]], package: str, version: str) -> str:
+    if python:
+        for specifier, py_ver in python.items():
+            specifier_set = SpecifierSet(specifier.replace(DEV_VERSION, DEV_NUMERIC))
+            if specifier_set.contains(DEV_NUMERIC if version == DEV_VERSION else version):
+                return py_ver
+
+    return get_requires_python(package, version)
+
+
 def remove_comments(s):
     return "\n".join(l for l in s.strip().split("\n") if not l.strip().startswith("#"))
 
@@ -366,6 +399,7 @@ def expand_config(config):
                 requirements.extend(get_matched_requirements(cfg.requirements or {}, str(ver)))
                 install = make_pip_install_command(requirements)
                 run = remove_comments(cfg.run)
+                python = get_python_version(cfg.python, package_info.pip_release, str(ver))
                 java = get_java_version(cfg.java, str(ver))
 
                 matrix.add(
@@ -378,6 +412,7 @@ def expand_config(config):
                         run=run,
                         package=package_info.pip_release,
                         version=ver,
+                        python=python,
                         java=java,
                         supported=ver <= cfg.maximum,
                         free_disk_space=free_disk_space,
@@ -392,6 +427,7 @@ def expand_config(config):
                     install = make_pip_install_command(requirements) + "\n" + install_dev
                 else:
                     install = install_dev
+                python = get_python_version(cfg.python, package_info.pip_release, DEV_VERSION)
                 java = get_java_version(cfg.java, DEV_VERSION)
 
                 run = remove_comments(cfg.run)
@@ -406,6 +442,7 @@ def expand_config(config):
                         run=run,
                         package=package_info.pip_release,
                         version=dev_version,
+                        python=python,
                         java=java,
                         supported=False,
                         free_disk_space=free_disk_space,

mlflow/ml-package-versions.yml

@@ -9,12 +9,16 @@ sklearn:
     maximum: "1.4.1.post1"
     requirements:
       "< 1.0": ["scipy==1.7.3"]
+    python:
+      "== dev": "3.9"
     run: |
       pytest tests/sklearn/test_sklearn_model_export.py
 
   autologging:
     minimum: "0.24.1"
     maximum: "1.4.1.post1"
+    python:
+      "== dev": "3.9"
     requirements:
       ">= 0.0.0": ["matplotlib"]
       "< 1.0": ["scipy==1.7.3"]
@@ -83,6 +87,8 @@ keras:
   models:
     minimum: "3.0.2"
     maximum: "3.0.5"
+    python:
+      "== dev": "3.9"
     requirements:
       ">= 3.0.0": ["jax[cpu]>0.4"]
     run: |
@@ -92,6 +98,8 @@ keras:
   autologging:
     minimum: "3.0.2"
     maximum: "3.0.5"
+    python:
+      "== dev": "3.9"
     requirements:
       ">= 3.0.0": ["jax[cpu]>0.4"]
     run: |
@@ -107,6 +115,8 @@ tensorflow:
   models:
     minimum: "2.6.5"
     maximum: "2.15.0.post1"
+    python:
+      "== dev": "3.9"
     requirements:
       # Requirements to run tests for keras
       ">= 0.0.0": ["scikit-learn", "pyspark", "pyarrow", "transformers!=4.38.0,!=4.38.1"]
@@ -132,6 +142,8 @@ tensorflow:
   autologging:
     minimum: "2.6.5"
     maximum: "2.15.0.post1"
+    python:
+      "== dev": "3.9"
     requirements:
       "== dev": ["scikit-learn"]
       "< 2.7.0": ["pandas==1.3.5"]
@@ -321,6 +333,8 @@ statsmodels:
   models:
     minimum: "0.11.1"
     maximum: "0.14.1"
+    python:
+      "== dev": "3.9"
     requirements:
       "< 0.13.0": ["pandas==1.3.5", "scipy==1.7.3"]
     run: |
@@ -329,6 +343,8 @@ statsmodels:
   autologging:
     minimum: "0.11.1"
     maximum: "0.14.1"
+    python:
+      "== dev": "3.9"
     requirements:
       "< 0.13.0": ["pandas==1.3.5", "scipy==1.7.3"]
     run: |