From 69f78e18d75e609fc719768337c77a19c128c848 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 1 Aug 2022 15:08:43 +0200 Subject: [PATCH] pkg: parse local versions (#13933) * pkg: parse local versions * offline * str * manifest * ci (cherry picked from commit 47833dfd1adddf0fea6421cd317d903ea49ebda0) --- .actions/setup_tools.py | 34 ++++++++++++++++++++++++++++++++-- requirements/base.txt | 2 ++ setup.py | 10 ++++++++++ src/lightning/__setup__.py | 29 ++++++++++------------------- 4 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 requirements/base.txt diff --git a/.actions/setup_tools.py b/.actions/setup_tools.py index 552da612588f6..6b35857de1217 100644 --- a/.actions/setup_tools.py +++ b/.actions/setup_tools.py @@ -22,7 +22,7 @@ import urllib.request from datetime import datetime from importlib.util import module_from_spec, spec_from_file_location -from itertools import groupby +from itertools import chain, groupby from types import ModuleType from typing import List @@ -45,7 +45,7 @@ def _load_py_module(name: str, location: str) -> ModuleType: def load_requirements( path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True ) -> List[str]: - """Load requirements from a file. + """Loading requirements from a file. >>> path_req = os.path.join(_PROJECT_ROOT, "requirements") >>> load_requirements(path_req) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE @@ -438,3 +438,33 @@ def _download_frontend(root: str = _PROJECT_ROOT): # If installing from source without internet connection, we don't want to break the installation except Exception: print("The Lightning UI downloading has failed!") + + +def _adjust_require_versions(source_dir: str = "src", req_dir: str = "requirements") -> None: + """Parse the base requirements and append as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`.""" + reqs = load_requirements(req_dir, file_name="base.txt") + for i, req in enumerate(reqs): + pkg_name = req[: min(req.index(c) for c in ">=" if c in req)] + ver_ = parse_version_from_file(os.path.join(source_dir, pkg_name)) + if not ver_: + continue + ver2 = ".".join(ver_.split(".")[:2] + ["*"]) + reqs[i] = f"{req}, =={ver2}" + + with open(os.path.join(req_dir, "base.txt"), "w") as fp: + fp.writelines([ln + os.linesep for ln in reqs]) + + +def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requirements: bool = False) -> None: + """Load all base requirements from all particular packages and prune duplicates.""" + requires = [ + load_requirements(d, file_name="base.txt", unfreeze=not freeze_requirements) + for d in glob.glob(os.path.join(req_dir, "*")) + if os.path.isdir(d) + ] + if not requires: + return None + # TODO: add some smarter version aggregation per each package + requires = list(chain(*requires)) + with open(os.path.join(req_dir, "base.txt"), "w") as fp: + fp.writelines([ln + os.linesep for ln in requires]) diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 0000000000000..fea4ee10f4ce7 --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,2 @@ +pytorch_lightning>=1.6.5 +lightning_app>=0.5.2 diff --git a/setup.py b/setup.py index 519829acee02e..7d4084960d450 100755 --- a/setup.py +++ b/setup.py @@ -60,7 +60,9 @@ # http://blog.ionelmc.ro/2014/05/25/python-packaging/ _PATH_ROOT = os.path.dirname(__file__) _PATH_SRC = os.path.join(_PATH_ROOT, "src") +_PATH_REQUIRE = os.path.join(_PATH_ROOT, "requirements") _PATH_SETUP = os.path.join(_PATH_SRC, _REAL_PKG_NAME or "lightning", "__setup__.py") +_FREEZE_REQUIREMENTS = bool(int(os.environ.get("FREEZE_REQUIREMENTS", 0))) # Hardcode the env variable from time of package creation, otherwise it fails during installation @@ -89,11 +91,19 @@ def _load_py_module(name: str, location: str) -> ModuleType: # engineer specific practices if __name__ == "__main__": _SETUP_TOOLS = _load_py_module(name="setup_tools", location=os.path.join(".actions", "setup_tools.py")) + + if _PACKAGE_NAME == "lightning": # install just the meta package + _SETUP_TOOLS._adjust_require_versions(_PATH_SRC, _PATH_REQUIRE) + elif _PACKAGE_NAME not in _PACKAGE_MAPPING: # install everything + _SETUP_TOOLS._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS) + if _PACKAGE_NAME not in _PACKAGE_MAPPING: _SETUP_TOOLS.set_version_today(os.path.join(_PATH_SRC, "lightning", "__version__.py")) + for lit_name, pkg_name in _PACKAGE_MAPPING.items(): # fixme: if we run creation of meta pkg against stable we shall pull the source _SETUP_TOOLS.create_meta_package(os.path.join(_PATH_ROOT, "src"), pkg_name, lit_name) + _SETUP_MODULE = _load_py_module(name="pkg_setup", location=_PATH_SETUP) _SETUP_MODULE._adjust_manifest(pkg_name=_REAL_PKG_NAME) setup(**_SETUP_MODULE._setup_args(pkg_name=_REAL_PKG_NAME)) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index ba0dc7ed8cd5d..e3ada2d7e93df 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -1,7 +1,5 @@ -import glob import os.path from importlib.util import module_from_spec, spec_from_file_location -from itertools import chain from types import ModuleType from typing import Any, Dict @@ -10,6 +8,7 @@ _PROJECT_ROOT = "." _SOURCE_ROOT = os.path.join(_PROJECT_ROOT, "src") _PACKAGE_ROOT = os.path.join(_SOURCE_ROOT, "lightning") +_PATH_REQUIREMENTS = os.path.join("requirements") _FREEZE_REQUIREMENTS = bool(int(os.environ.get("FREEZE_REQUIREMENTS", 0))) @@ -22,6 +21,9 @@ def _load_py_module(name: str, location: str) -> ModuleType: return py +_SETUP_TOOLS = _load_py_module("setup_tools", os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")) + + def _adjust_manifest(**kwargs: Any) -> None: # todo: consider rather aggregation of particular manifest adjustments manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in") @@ -31,6 +33,7 @@ def _adjust_manifest(**kwargs: Any) -> None: if kwargs["pkg_name"] == "lightning": lines += [ "recursive-include src/lightning *.md", + "include requirements/base.txt", # fixme: this is strange, this shall work with setup find package - include "prune src/lightning_app", "prune src/pytorch_lightning", @@ -47,29 +50,17 @@ def _adjust_manifest(**kwargs: Any) -> None: def _setup_args(**kwargs: Any) -> Dict[str, Any]: - _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py") - _setup_tools = _load_py_module("setup_tools", _path_setup_tools) _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py")) _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py")) - _long_description = _setup_tools.load_readme_description( + _long_description = _SETUP_TOOLS.load_readme_description( _PROJECT_ROOT, homepage=_about.__homepage__, version=_version.version ) - if kwargs["pkg_name"] == "lightning": - _include_pkgs = ["lightning", "lightning.*"] - # todo: generate this list automatically with parsing feature pkg versions - _requires = ["pytorch-lightning>=1.6.5, <1.7.0", "lightning-app>=0.5.2, <0.6.0"] - else: - _include_pkgs = ["*"] - _requires = [ - _setup_tools.load_requirements(d, unfreeze=not _FREEZE_REQUIREMENTS) - for d in glob.glob(os.path.join("requirements", "*")) - if os.path.isdir(d) - ] - _requires = list(chain(*_requires)) + _include_pkgs = ["lightning", "lightning.*"] if kwargs["pkg_name"] == "lightning" else ["*"] + # TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip # TODO: remove this once lightning-ui package is ready as a dependency - _setup_tools._download_frontend(_PROJECT_ROOT) + _SETUP_TOOLS._download_frontend(_PROJECT_ROOT) return dict( name="lightning", @@ -94,7 +85,7 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]: ], }, setup_requires=[], - install_requires=_requires, + install_requires=_SETUP_TOOLS.load_requirements(_PATH_REQUIREMENTS, unfreeze=True), extras_require={}, # todo: consider porting all other packages extras with prefix project_urls={ "Bug Tracker": "https://github.com/Lightning-AI/lightning/issues",