From 61ee3fabc3ca4fbc63d5c294c0b3f91f686d23cd Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Sat, 12 Nov 2022 16:36:36 +0100 Subject: [PATCH] PKG: distribute single semver (#15374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * global * distrib ver * codeowners * Apply suggestions from code review Co-authored-by: otaj <6065855+otaj@users.noreply.github.com> Co-authored-by: Jirka Borovec Co-authored-by: Carlos Mocholí --- .actions/setup_tools.py | 11 +++++++++++ .github/actions/pkg-check/action.yml | 3 ++- .github/workflows/docs-checks.yml | 3 +++ .gitignore | 1 + MANIFEST.in | 1 + dockers/nvidia/Dockerfile | 7 ++++--- dockers/release/Dockerfile | 6 ++++-- setup.py | 8 ++++++-- src/lightning/__setup__.py | 3 ++- src/lightning/__version__.py | 10 +++++++++- src/lightning_app/__setup__.py | 3 ++- src/lightning_app/__version__.py | 10 +++++++++- src/lightning_lite/__setup__.py | 3 ++- src/lightning_lite/__version__.py | 10 +++++++++- src/pytorch_lightning/__setup__.py | 4 +++- src/pytorch_lightning/__version__.py | 10 +++++++++- src/version.info | 1 + 17 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 src/version.info diff --git a/.actions/setup_tools.py b/.actions/setup_tools.py index a91a6681a7383..0f05c270c62f6 100644 --- a/.actions/setup_tools.py +++ b/.actions/setup_tools.py @@ -151,6 +151,17 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str: return text +def distribute_version(src_folder: str, ver_file: str = "version.info") -> None: + """Copy the global version to all packages.""" + ls_ver = glob.glob(os.path.join(src_folder, "*", "__version__.py")) + ver_template = os.path.join(src_folder, ver_file) + for fpath in ls_ver: + fpath = os.path.join(os.path.dirname(fpath), ver_file) + if os.path.isfile(fpath): + os.remove(fpath) + shutil.copy2(ver_template, fpath) + + def _download_frontend(pkg_path: str): """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct directory.""" diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml index 6680f945d589d..c325113d787af 100644 --- a/.github/actions/pkg-check/action.yml +++ b/.github/actions/pkg-check/action.yml @@ -50,7 +50,8 @@ runs: run: | import os, glob, pathlib, shutil # list folders without ending .egg-info - dirs = [d for d in glob.glob(os.path.join("*", "src", "*")) if not d.endswith(".egg-info")] + ls = glob.glob(os.path.join("*", "src", "*")) + dirs = [d for d in ls if os.path.isdir(d) and not d.endswith(".egg-info")] print(dirs) assert len(dirs) == ${{ inputs.nb-dirs }} # cleaning diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index c831a35f8f022..3072e32e8a9f6 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -21,6 +21,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} +env: + FREEZE_REQUIREMENTS: "1" + defaults: run: shell: bash diff --git a/.gitignore b/.gitignore index 054a5ba16aff5..982309a8a356a 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ wheels/ .installed.cfg *.egg src/lightning/*/ +src/*/version.info # PyInstaller # Usually these files are written by a python script from a template diff --git a/MANIFEST.in b/MANIFEST.in index 10af40c3dd1cf..ac8c2556d4f02 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,5 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include .actions/assistant.py +include src/version.info include *.cff # citation info diff --git a/dockers/nvidia/Dockerfile b/dockers/nvidia/Dockerfile index 5e67d2044f46d..4b97c8a32b88f 100644 --- a/dockers/nvidia/Dockerfile +++ b/dockers/nvidia/Dockerfile @@ -42,9 +42,10 @@ RUN \ cp -r lightning/*examples . && \ # Installations \ - pip install -q fire && \ pip install "Pillow>=8.2, !=8.3.0" "cryptography>=3.4" "py>=1.10" --no-cache-dir && \ - pip install ./lightning["extra","loggers","strategies","examples"] --no-cache-dir && \ + # remove colossalai from requirements since they are installed separately + python -c "fname = 'lightning/requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)" ; \ + PACKAGE_NAME=pytorch pip install './lightning[extra,loggers,strategies]' --no-cache-dir && \ rm -rf lightning && \ pip list @@ -60,6 +61,6 @@ RUN \ pip --version && \ pip list | grep torch && \ python -c "import torch; assert torch.__version__.startswith('$TORCH_VERSION'), torch.__version__" && \ - python -c "import lightning as L; print(L.__version__)" + python -c "import pytorch_lightning as pl; print(pl.__version__)" CMD ["jupyter", "notebook", "--port=8888", "--no-browser", "--ip=0.0.0.0", "--allow-root"] diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile index 2ab2f88fe552c..cee1e8b381fc6 100644 --- a/dockers/release/Dockerfile +++ b/dockers/release/Dockerfile @@ -39,13 +39,15 @@ RUN \ fi && \ # otherwise there is collision with folder name ans pkg name on Pypi cd lightning && \ - pip install .["extra","loggers","strategies"] --no-cache-dir --find-links https://release.colossalai.org && \ + # remove colossalai from requirements since they are installed separately + python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)" ; \ + PACKAGE_NAME=pytorch pip install '.[extra,loggers,strategies]' --no-cache-dir && \ cd .. && \ rm -rf lightning RUN python --version && \ pip --version && \ pip list && \ - python -c "import lightning as L; print(L.__version__)" + python -c "import pytorch_lightning as pl; print(pl.__version__)" # CMD ["/bin/bash"] diff --git a/setup.py b/setup.py index 83a76b134c54b..dae47e77b0c25 100755 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ 3. Building packages as sdist or binary wheel and installing or publish to PyPI afterwords you use command `python setup.py sdist` or `python setup.py bdist_wheel` accordingly. - In case you want to build just a particular package you would use exporting env. variable as above: - `export PACKAGE_NAME=pytorch|app|lite ; python setup.py sdist bdist_wheel` + In case you want to build just a particular package you want to set an environment variable: + `PACKAGE_NAME=lightning|pytorch|app|lite python setup.py sdist|bdist_wheel` 4. Automated releasing with GitHub action is natural extension of 3) is composed of three consecutive steps: a) determine which packages shall be released based on version increment in `__version__.py` and eventually @@ -73,6 +73,10 @@ def _load_py_module(name: str, location: str) -> ModuleType: setup_tools = _load_py_module(name="setup_tools", location=os.path.join(_PATH_ROOT, ".actions", "setup_tools.py")) assistant = _load_py_module(name="assistant", location=os.path.join(_PATH_ROOT, ".actions", "assistant.py")) + if os.path.exists(_PATH_SRC): # not a wheel install + # copy the version information to all packages + setup_tools.distribute_version(_PATH_SRC) + package_to_install = _PACKAGE_NAME or "lightning" print(f"Installing the {package_to_install} package") # requires `-v` to appear if package_to_install == "lightning": # install everything diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 6f30e218ab6e5..87d0320b7c011 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -35,6 +35,7 @@ def _adjust_manifest(**kwargs: Any) -> None: "recursive-include requirements *.txt", "recursive-include src/lightning/app/ui *", "recursive-include src/lightning/cli/*-template *", # Add templates as build-in + "include src/lightning/version.info" + os.linesep, "include src/lightning/app/components/serve/catimage.png" + os.linesep, # fixme: this is strange, this shall work with setup find package - include "prune src/lightning_app", @@ -60,7 +61,7 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]: return dict( name="lightning", - version=_version.version, # todo: consider adding branch for installation from source + version=_version.version, description=_about.__docs__, author=_about.__author__, author_email=_about.__author_email__, diff --git a/src/lightning/__version__.py b/src/lightning/__version__.py index 0bf4a385a2903..ec7becccbcf5e 100644 --- a/src/lightning/__version__.py +++ b/src/lightning/__version__.py @@ -1 +1,9 @@ -version = "1.9.0dev" +import os + +_PACKAGE_ROOT = os.path.dirname(__file__) +_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info") +if not os.path.exists(_SDIST_PATH): + # relevant for `bdist_wheel` + _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info") +with open(_VERSION_PATH, encoding="utf-8") as fo: + version = fo.readlines()[0].strip() diff --git a/src/lightning_app/__setup__.py b/src/lightning_app/__setup__.py index 7a649af448a05..f6530ca869d68 100644 --- a/src/lightning_app/__setup__.py +++ b/src/lightning_app/__setup__.py @@ -53,6 +53,7 @@ def _adjust_manifest(**__: Any) -> None: "include src/lightning_app/components/serve/catimage.png" + os.linesep, "recursive-include requirements/app *.txt" + os.linesep, "recursive-include src/lightning_app/cli/*-template *" + os.linesep, # Add templates + "include src/lightning_app/version.info" + os.linesep, ] # TODO: remove this once lightning-ui package is ready as a dependency @@ -76,7 +77,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]: return dict( name="lightning-app", - version=_version.version, # todo: consider using date version + branch for installation from source + version=_version.version, description=_about.__docs__, author=_about.__author__, author_email=_about.__author_email__, diff --git a/src/lightning_app/__version__.py b/src/lightning_app/__version__.py index 0bf4a385a2903..ec7becccbcf5e 100644 --- a/src/lightning_app/__version__.py +++ b/src/lightning_app/__version__.py @@ -1 +1,9 @@ -version = "1.9.0dev" +import os + +_PACKAGE_ROOT = os.path.dirname(__file__) +_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info") +if not os.path.exists(_SDIST_PATH): + # relevant for `bdist_wheel` + _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info") +with open(_VERSION_PATH, encoding="utf-8") as fo: + version = fo.readlines()[0].strip() diff --git a/src/lightning_lite/__setup__.py b/src/lightning_lite/__setup__.py index b00cedb83e5b4..6b76436df7e37 100644 --- a/src/lightning_lite/__setup__.py +++ b/src/lightning_lite/__setup__.py @@ -52,6 +52,7 @@ def _adjust_manifest(**__: Any) -> None: "recursive-exclude requirements *.txt" + os.linesep, "recursive-include requirements/lite *.txt" + os.linesep, "recursive-include src/lightning_lite *.md" + os.linesep, + "include src/lightning_lite/version.info" + os.linesep, ] # TODO: remove this once lightning-ui package is ready as a dependency @@ -72,7 +73,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]: return dict( name="lightning-lite", - version=_version.version, # todo: consider using date version + branch for installation from source + version=_version.version, description=_about.__docs__, author=_about.__author__, author_email=_about.__author_email__, diff --git a/src/lightning_lite/__version__.py b/src/lightning_lite/__version__.py index 0bf4a385a2903..ec7becccbcf5e 100644 --- a/src/lightning_lite/__version__.py +++ b/src/lightning_lite/__version__.py @@ -1 +1,9 @@ -version = "1.9.0dev" +import os + +_PACKAGE_ROOT = os.path.dirname(__file__) +_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info") +if not os.path.exists(_SDIST_PATH): + # relevant for `bdist_wheel` + _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info") +with open(_VERSION_PATH, encoding="utf-8") as fo: + version = fo.readlines()[0].strip() diff --git a/src/pytorch_lightning/__setup__.py b/src/pytorch_lightning/__setup__.py index 442bda630b884..6035b8dc4b869 100644 --- a/src/pytorch_lightning/__setup__.py +++ b/src/pytorch_lightning/__setup__.py @@ -54,8 +54,10 @@ def _adjust_manifest(**__: Any) -> None: "recursive-exclude requirements *.txt" + os.linesep, "recursive-include requirements/lite *.txt" + os.linesep, "recursive-include src/lightning_lite *.md" + os.linesep, + "include src/lightning_lite/version.info" + os.linesep, "recursive-include src/pytorch_lightning *.md" + os.linesep, "recursive-include requirements/pytorch *.txt" + os.linesep, + "include src/pytorch_lightning/version.info" + os.linesep, "include src/pytorch_lightning/py.typed" + os.linesep, # marker file for PEP 561 ] with open(manifest_path, "w") as fp: @@ -72,7 +74,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]: ) return dict( name="pytorch-lightning", - version=_version.version, # todo: consider using date version + branch for installation from source + version=_version.version, description=_about.__docs__, author=_about.__author__, author_email=_about.__author_email__, diff --git a/src/pytorch_lightning/__version__.py b/src/pytorch_lightning/__version__.py index 0bf4a385a2903..ec7becccbcf5e 100644 --- a/src/pytorch_lightning/__version__.py +++ b/src/pytorch_lightning/__version__.py @@ -1 +1,9 @@ -version = "1.9.0dev" +import os + +_PACKAGE_ROOT = os.path.dirname(__file__) +_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info") +if not os.path.exists(_SDIST_PATH): + # relevant for `bdist_wheel` + _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info") +with open(_VERSION_PATH, encoding="utf-8") as fo: + version = fo.readlines()[0].strip() diff --git a/src/version.info b/src/version.info new file mode 100644 index 0000000000000..31662c9819f8b --- /dev/null +++ b/src/version.info @@ -0,0 +1 @@ +1.9.0dev