From b9858fcf03d21132f17625c89b6dbc91a2dae4c9 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 23 Nov 2022 16:39:06 +0100 Subject: [PATCH 01/10] lit extras --- src/lightning/__setup__.py | 40 +++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 3656896c7b3ac..d8e8824514f30 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -1,3 +1,4 @@ +import glob import os.path from importlib.util import module_from_spec, spec_from_file_location from types import ModuleType @@ -5,10 +6,10 @@ from setuptools import find_packages -_PROJECT_ROOT = "." +_PROJECT_ROOT = "../.." # FIXME _SOURCE_ROOT = os.path.join(_PROJECT_ROOT, "src") _PACKAGE_ROOT = os.path.join(_SOURCE_ROOT, "lightning") -_PATH_REQUIREMENTS = os.path.join("requirements") +_PATH_REQUIREMENTS = os.path.join(_PROJECT_ROOT, "requirements") _FREEZE_REQUIREMENTS = bool(int(os.environ.get("FREEZE_REQUIREMENTS", 0))) @@ -24,6 +25,39 @@ def _load_py_module(name: str, location: str) -> ModuleType: _SETUP_TOOLS = _load_py_module("setup_tools", os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")) +def _prepare_extras() -> Dict[str, Any]: + # https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras + # Define package extras. These are only installed if you specify them. + # From remote, use like `pip install pytorch-lightning[dev, docs]` + # From local copy of repo, use like `pip install ".[dev, docs]"` + req_files = glob.glob(os.path.join(_PATH_REQUIREMENTS, "*", "*.txt")) + reqs = [dict(dir=os.path.dirname(p), fname=os.path.basename(p)) for p in req_files] + common_args = dict(unfreeze="major" if _FREEZE_REQUIREMENTS else "all") + reqs = [ + dict( + dir=r["dir"], + fname=r["fname"], + extra=os.path.splitext(r["fname"])[0], + group=os.path.basename(r["dir"]), + req=_SETUP_TOOLS.load_requirements(file_name=r["fname"], path_dir=r["dir"], **common_args), + ) + for r in reqs + if r["fname"] not in ("docs.txt", "devel.txt", "base.txt") + ] + + extras = {f"{r['group']}_{r['extra']}": r["req"] for r in reqs} + extra_names = list(extras.keys()) + for extra in extra_names: + name = "_".join(extra.split("_")[1:]) + extras[name] = extras.get(name, []) + extras[extra] + # todo + # extras["extra"] = extras["cloud"] + extras["ui"] + # extras["dev"] = extras["extra"] + extras["test"] # + extras['docs'] + # extras["all"] = extras["dev"] + extras = {name: list(set(reqs)) for name, reqs in extras.items()} + return extras + + def _adjust_manifest(**kwargs: Any) -> None: # todo: consider rather aggregation of particular manifest adjustments manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in") @@ -83,7 +117,7 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]: }, setup_requires=[], install_requires=_SETUP_TOOLS.load_requirements(_PATH_REQUIREMENTS, unfreeze="all"), - extras_require={}, # todo: consider porting all other packages extras with prefix + extras_require=_prepare_extras(), project_urls={ "Bug Tracker": "https://github.com/Lightning-AI/lightning/issues", "Documentation": "https://lightning.ai/lightning-docs", From 3f4e1722fd4a71ba1d6b19e007f7f5c932c59102 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 23 Nov 2022 17:01:08 +0100 Subject: [PATCH 02/10] . --- src/lightning/__setup__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index d8e8824514f30..08ad3eed1caa5 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -6,7 +6,7 @@ from setuptools import find_packages -_PROJECT_ROOT = "../.." # FIXME +_PROJECT_ROOT = "." _SOURCE_ROOT = os.path.join(_PROJECT_ROOT, "src") _PACKAGE_ROOT = os.path.join(_SOURCE_ROOT, "lightning") _PATH_REQUIREMENTS = os.path.join(_PROJECT_ROOT, "requirements") From cb483dc0975840ae35b477f782c0cf4f2342367d Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 23 Nov 2022 17:09:45 +0100 Subject: [PATCH 03/10] update --- src/lightning/__setup__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 08ad3eed1caa5..e046fba05a283 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -1,6 +1,7 @@ import glob import os.path from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path from types import ModuleType from typing import Any, Dict @@ -30,19 +31,18 @@ def _prepare_extras() -> Dict[str, Any]: # Define package extras. These are only installed if you specify them. # From remote, use like `pip install pytorch-lightning[dev, docs]` # From local copy of repo, use like `pip install ".[dev, docs]"` - req_files = glob.glob(os.path.join(_PATH_REQUIREMENTS, "*", "*.txt")) - reqs = [dict(dir=os.path.dirname(p), fname=os.path.basename(p)) for p in req_files] + req_files = [Path(p) for p in glob.glob(os.path.join(_PATH_REQUIREMENTS, "*", "*.txt"))] common_args = dict(unfreeze="major" if _FREEZE_REQUIREMENTS else "all") reqs = [ dict( - dir=r["dir"], - fname=r["fname"], - extra=os.path.splitext(r["fname"])[0], - group=os.path.basename(r["dir"]), - req=_SETUP_TOOLS.load_requirements(file_name=r["fname"], path_dir=r["dir"], **common_args), + dir=p.parent, + fname=p.name, + extra=p.stem, + group=p.parent.name, + req=_SETUP_TOOLS.load_requirements(file_name=p.name, path_dir=p.parent, **common_args), ) - for r in reqs - if r["fname"] not in ("docs.txt", "devel.txt", "base.txt") + for p in req_files + if p.name not in ("docs.txt", "devel.txt", "base.txt") ] extras = {f"{r['group']}_{r['extra']}": r["req"] for r in reqs} From 06d8783493578d1bf51d2011c01fa23ac468e69c Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Wed, 23 Nov 2022 17:20:23 +0100 Subject: [PATCH 04/10] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Carlos Mocholí --- src/lightning/__setup__.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index e046fba05a283..ca84c3b891e8a 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -34,20 +34,15 @@ def _prepare_extras() -> Dict[str, Any]: req_files = [Path(p) for p in glob.glob(os.path.join(_PATH_REQUIREMENTS, "*", "*.txt"))] common_args = dict(unfreeze="major" if _FREEZE_REQUIREMENTS else "all") reqs = [ - dict( - dir=p.parent, - fname=p.name, - extra=p.stem, - group=p.parent.name, - req=_SETUP_TOOLS.load_requirements(file_name=p.name, path_dir=p.parent, **common_args), - ) + { + f"{p.parent.name}_{p.stem}": _SETUP_TOOLS.load_requirements( + file_name=p.name, path_dir=p.parent, **common_args + ) + } for p in req_files if p.name not in ("docs.txt", "devel.txt", "base.txt") ] - - extras = {f"{r['group']}_{r['extra']}": r["req"] for r in reqs} - extra_names = list(extras.keys()) - for extra in extra_names: + for extra in extras: name = "_".join(extra.split("_")[1:]) extras[name] = extras.get(name, []) + extras[extra] # todo From f2157d01bf73963dd0b976e578abaf3e443c4b4d Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 23 Nov 2022 17:21:56 +0100 Subject: [PATCH 05/10] fix --- src/lightning/__setup__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index ca84c3b891e8a..192f6570ab192 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -33,15 +33,11 @@ def _prepare_extras() -> Dict[str, Any]: # From local copy of repo, use like `pip install ".[dev, docs]"` req_files = [Path(p) for p in glob.glob(os.path.join(_PATH_REQUIREMENTS, "*", "*.txt"))] common_args = dict(unfreeze="major" if _FREEZE_REQUIREMENTS else "all") - reqs = [ - { - f"{p.parent.name}_{p.stem}": _SETUP_TOOLS.load_requirements( - file_name=p.name, path_dir=p.parent, **common_args - ) - } + extras = { + f"{p.parent.name}_{p.stem}": _SETUP_TOOLS.load_requirements(file_name=p.name, path_dir=p.parent, **common_args) for p in req_files if p.name not in ("docs.txt", "devel.txt", "base.txt") - ] + } for extra in extras: name = "_".join(extra.split("_")[1:]) extras[name] = extras.get(name, []) + extras[extra] From 7ee3128a0a1ecdf06715cbb41d95238a8ed930cc Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 23 Nov 2022 17:23:52 +0100 Subject: [PATCH 06/10] fix2 --- src/lightning/__setup__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 192f6570ab192..bfe51de2f3d61 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -38,7 +38,8 @@ def _prepare_extras() -> Dict[str, Any]: for p in req_files if p.name not in ("docs.txt", "devel.txt", "base.txt") } - for extra in extras: + extra_names = list(extras.keys()) + for extra in extra_names: name = "_".join(extra.split("_")[1:]) extras[name] = extras.get(name, []) + extras[extra] # todo From fd7e66e28278a29efc248b48c4422b5d458e42b5 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 23 Nov 2022 17:37:41 +0100 Subject: [PATCH 07/10] - --- src/lightning/__setup__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index bfe51de2f3d61..5f51e0fc6bf6a 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -34,13 +34,13 @@ def _prepare_extras() -> Dict[str, Any]: req_files = [Path(p) for p in glob.glob(os.path.join(_PATH_REQUIREMENTS, "*", "*.txt"))] common_args = dict(unfreeze="major" if _FREEZE_REQUIREMENTS else "all") extras = { - f"{p.parent.name}_{p.stem}": _SETUP_TOOLS.load_requirements(file_name=p.name, path_dir=p.parent, **common_args) + f"{p.parent.name}-{p.stem}": _SETUP_TOOLS.load_requirements(file_name=p.name, path_dir=p.parent, **common_args) for p in req_files if p.name not in ("docs.txt", "devel.txt", "base.txt") } extra_names = list(extras.keys()) for extra in extra_names: - name = "_".join(extra.split("_")[1:]) + name = "-".join(extra.split("-")[1:]) extras[name] = extras.get(name, []) + extras[extra] # todo # extras["extra"] = extras["cloud"] + extras["ui"] From c7e0bfa781b6ca103b72ad719b635eb47bb672e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Wed, 23 Nov 2022 19:31:50 +0100 Subject: [PATCH 08/10] List copy --- src/lightning/__setup__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 5f51e0fc6bf6a..a43ba7bee8f65 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -38,8 +38,7 @@ def _prepare_extras() -> Dict[str, Any]: for p in req_files if p.name not in ("docs.txt", "devel.txt", "base.txt") } - extra_names = list(extras.keys()) - for extra in extra_names: + for extra in list(extras): name = "-".join(extra.split("-")[1:]) extras[name] = extras.get(name, []) + extras[extra] # todo From 163b648a18150966bc622d8f80a1dae101761018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Wed, 23 Nov 2022 19:32:16 +0100 Subject: [PATCH 09/10] Verbose print --- .github/workflows/docs-checks.yml | 2 +- src/lightning/__setup__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 01d50bd61eb8c..98a14d70e792d 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -79,7 +79,7 @@ jobs: # This is needed as App docs is heavily using/referring to lightning package if: ${{ matrix.pkg-name == 'app' }} run: | - pip install -e . -U -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi + pip install -e . -U -v -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi git checkout -- . - name: Adjust docs refs diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index a43ba7bee8f65..bc3d2c1afe94b 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -46,6 +46,7 @@ def _prepare_extras() -> Dict[str, Any]: # extras["dev"] = extras["extra"] + extras["test"] # + extras['docs'] # extras["all"] = extras["dev"] extras = {name: list(set(reqs)) for name, reqs in extras.items()} + print("The extras are", extras) return extras From e3832735371fb46b974007883381e8f350c22a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Thu, 24 Nov 2022 18:01:00 +0100 Subject: [PATCH 10/10] Somehow this is breaking the build --- requirements/pytorch/strategies.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/pytorch/strategies.txt b/requirements/pytorch/strategies.txt index 2f0cce54f4158..040b8212776b9 100644 --- a/requirements/pytorch/strategies.txt +++ b/requirements/pytorch/strategies.txt @@ -6,4 +6,4 @@ fairscale>=0.4.5, <=0.4.6 deepspeed>=0.6.0, <=0.7.0 # no need to install with [pytorch] as pytorch is already installed horovod>=0.21.2, !=0.24.0, <=0.26.1 -hivemind>=1.0.1, <=1.0.1; sys_platform == 'linux' +hivemind==1.0.1; sys_platform == 'linux'