From 41f318868fb725b14896991061118709f1077f1c Mon Sep 17 00:00:00 2001 From: Harutaka Kawamura Date: Fri, 15 Mar 2024 17:50:14 +0900 Subject: [PATCH] Include python version in test matrix for cross version tests (#11433) Signed-off-by: harupy <17039389+harupy@users.noreply.github.com> --- .github/workflows/cross-version-tests.yml | 7 +-- dev/get_minimum_required_python.py | 70 ----------------------- dev/set_matrix.py | 47 +++++++++++++-- mlflow/ml-package-versions.yml | 16 ++++++ 4 files changed, 59 insertions(+), 81 deletions(-) delete mode 100644 dev/get_minimum_required_python.py diff --git a/.github/workflows/cross-version-tests.yml b/.github/workflows/cross-version-tests.yml index ebdaaeb5eb337d..1126883e168b04 100644 --- a/.github/workflows/cross-version-tests.yml +++ b/.github/workflows/cross-version-tests.yml @@ -133,14 +133,9 @@ jobs: ref: ${{ github.event.inputs.ref }} - uses: ./.github/actions/free-disk-space if: matrix.free_disk_space - - name: Get python version - id: get-python-version - run: | - python_version=$(python dev/get_minimum_required_python.py -p ${{ matrix.package }} -v ${{ matrix.version }} --python-versions "3.8,3.9") - echo "version=$python_version" >> $GITHUB_OUTPUT - uses: ./.github/actions/setup-python with: - python-version: ${{ steps.get-python-version.outputs.version }} + python-version: ${{ matrix.python }} - uses: ./.github/actions/setup-pyenv - uses: ./.github/actions/setup-java with: diff --git a/dev/get_minimum_required_python.py b/dev/get_minimum_required_python.py deleted file mode 100644 index 0645b32acf02c0..00000000000000 --- a/dev/get_minimum_required_python.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -A script to automatically find the minimum required python version for a specified package. - -Usage: -python dev/get_minimum_required_python.py -p scikit-learn -v 1.1.0 --python-versions "3.8" -""" -import argparse -import typing as t - -import requests -from packaging.specifiers import SpecifierSet -from packaging.version import Version - -# "dev" versions are installed from source so we can't programmatically get the required python. -_DEV_PACKAGES_PYTHON_VERSIONS = { - "tensorflow": ">=3.9", - "scikit-learn": ">=3.9", - "statsmodels": ">=3.9", - "keras": ">=3.9", -} - - -def get_requires_python(package: str, version: str) -> t.Optional[str]: - if version == "dev" and package in _DEV_PACKAGES_PYTHON_VERSIONS: - return _DEV_PACKAGES_PYTHON_VERSIONS[package] - - resp = requests.get(f"https://pypi.python.org/pypi/{package}/json") - resp.raise_for_status() - return next( - ( - distributions[0].get("requires_python") - for ver, distributions in resp.json()["releases"].items() - if ver == version and distributions - ), - None, - ) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("-p", "--package", help="Package name", required=True) - parser.add_argument("-v", "--version", help="Package version", required=True) - parser.add_argument( - "--python-versions", - help=( - "Comma separated string representing python versions. " - "If `requires_python` is unavailable for a specified package, " - "the minimum version will be selected." - ), - required=True, - ) - return parser.parse_args() - - -def main(): - args = parse_args() - sorted_python_versions = sorted(args.python_versions.split(","), key=Version) - min_python_version = sorted_python_versions[0] - requires_python = get_requires_python(args.package, args.version) - if not requires_python: - print(min_python_version) - return - - specifier_set = SpecifierSet(requires_python) - matched_versions = list(filter(specifier_set.contains, sorted_python_versions)) - print(matched_versions[0] if matched_versions else min_python_version) - - -if __name__ == "__main__": - main() diff --git a/dev/set_matrix.py b/dev/set_matrix.py index 7c6885c5e195de..0a2034996ab201 100644 --- a/dev/set_matrix.py +++ b/dev/set_matrix.py @@ -70,6 +70,7 @@ class TestConfig(BaseModel): maximum: Version unsupported: t.Optional[t.List[Version]] = None requirements: t.Optional[t.Dict[str, t.List[str]]] = None + python: t.Optional[t.Dict[str, str]] = None java: t.Optional[t.Dict[str, str]] = None run: str allow_unreleased_max_version: t.Optional[bool] = None @@ -100,6 +101,7 @@ class MatrixItem(BaseModel): run: str package: str version: Version + python: str java: str supported: bool free_disk_space: bool @@ -128,12 +130,8 @@ def read_yaml(location, if_error=None): raise -@functools.lru_cache def get_released_versions(package_name): - url = f"https://pypi.org/pypi/{package_name}/json" - response = requests.get(url) - response.raise_for_status() - data = response.json() + data = pypi_json(package_name) versions = [] for version, distributions in data["releases"].items(): if len(distributions) == 0 or any(d.get("yanked", False) for d in distributions): @@ -246,6 +244,41 @@ def get_java_version(java: t.Optional[t.Dict[str, str]], version: str) -> str: return default +@functools.lru_cache(maxsize=128) +def pypi_json(package: str) -> t.Dict[str, t.Any]: + resp = requests.get(f"https://pypi.org/pypi/{package}/json") + resp.raise_for_status() + return resp.json() + + +def get_requires_python(package: str, version: str) -> str: + package_json = pypi_json(package) + requires_python = next( + ( + distributions[0].get("requires_python") + for ver, distributions in package_json["releases"].items() + if ver == version and distributions + ), + None, + ) + candidates = ("3.8", "3.9") + if requires_python is None: + return candidates[0] + + spec = SpecifierSet(requires_python) + return next((c for c in candidates if spec.contains(c)), None) or candidates[0] + + +def get_python_version(python: t.Optional[t.Dict[str, str]], package: str, version: str) -> str: + if python: + for specifier, py_ver in python.items(): + specifier_set = SpecifierSet(specifier.replace(DEV_VERSION, DEV_NUMERIC)) + if specifier_set.contains(DEV_NUMERIC if version == DEV_VERSION else version): + return py_ver + + return get_requires_python(package, version) + + def remove_comments(s): return "\n".join(l for l in s.strip().split("\n") if not l.strip().startswith("#")) @@ -366,6 +399,7 @@ def expand_config(config): requirements.extend(get_matched_requirements(cfg.requirements or {}, str(ver))) install = make_pip_install_command(requirements) run = remove_comments(cfg.run) + python = get_python_version(cfg.python, package_info.pip_release, str(ver)) java = get_java_version(cfg.java, str(ver)) matrix.add( @@ -378,6 +412,7 @@ def expand_config(config): run=run, package=package_info.pip_release, version=ver, + python=python, java=java, supported=ver <= cfg.maximum, free_disk_space=free_disk_space, @@ -392,6 +427,7 @@ def expand_config(config): install = make_pip_install_command(requirements) + "\n" + install_dev else: install = install_dev + python = get_python_version(cfg.python, package_info.pip_release, DEV_VERSION) java = get_java_version(cfg.java, DEV_VERSION) run = remove_comments(cfg.run) @@ -406,6 +442,7 @@ def expand_config(config): run=run, package=package_info.pip_release, version=dev_version, + python=python, java=java, supported=False, free_disk_space=free_disk_space, diff --git a/mlflow/ml-package-versions.yml b/mlflow/ml-package-versions.yml index 6f31fa349a0036..7d5f7c194e53ea 100644 --- a/mlflow/ml-package-versions.yml +++ b/mlflow/ml-package-versions.yml @@ -9,12 +9,16 @@ sklearn: maximum: "1.4.1.post1" requirements: "< 1.0": ["scipy==1.7.3"] + python: + "== dev": "3.9" run: | pytest tests/sklearn/test_sklearn_model_export.py autologging: minimum: "0.24.1" maximum: "1.4.1.post1" + python: + "== dev": "3.9" requirements: ">= 0.0.0": ["matplotlib"] "< 1.0": ["scipy==1.7.3"] @@ -83,6 +87,8 @@ keras: models: minimum: "3.0.2" maximum: "3.0.5" + python: + "== dev": "3.9" requirements: ">= 3.0.0": ["jax[cpu]>0.4"] run: | @@ -92,6 +98,8 @@ keras: autologging: minimum: "3.0.2" maximum: "3.0.5" + python: + "== dev": "3.9" requirements: ">= 3.0.0": ["jax[cpu]>0.4"] run: | @@ -107,6 +115,8 @@ tensorflow: models: minimum: "2.6.5" maximum: "2.15.0.post1" + python: + "== dev": "3.9" requirements: # Requirements to run tests for keras ">= 0.0.0": ["scikit-learn", "pyspark", "pyarrow", "transformers!=4.38.0,!=4.38.1"] @@ -132,6 +142,8 @@ tensorflow: autologging: minimum: "2.6.5" maximum: "2.15.0.post1" + python: + "== dev": "3.9" requirements: "== dev": ["scikit-learn"] "< 2.7.0": ["pandas==1.3.5"] @@ -321,6 +333,8 @@ statsmodels: models: minimum: "0.11.1" maximum: "0.14.1" + python: + "== dev": "3.9" requirements: "< 0.13.0": ["pandas==1.3.5", "scipy==1.7.3"] run: | @@ -329,6 +343,8 @@ statsmodels: autologging: minimum: "0.11.1" maximum: "0.14.1" + python: + "== dev": "3.9" requirements: "< 0.13.0": ["pandas==1.3.5", "scipy==1.7.3"] run: |