PKG: distribute single semver (#15374)

* global
* distrib ver
* codeowners
* Apply suggestions from code review

Co-authored-by: otaj <[email protected]>
Co-authored-by: Jirka Borovec <[email protected]>
Co-authored-by: Carlos Mocholí <[email protected]>

.actions/setup_tools.py

@@ -151,6 +151,17 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
     return text
 
 
+def distribute_version(src_folder: str, ver_file: str = "version.info") -> None:
+    """Copy the global version to all packages."""
+    ls_ver = glob.glob(os.path.join(src_folder, "*", "__version__.py"))
+    ver_template = os.path.join(src_folder, ver_file)
+    for fpath in ls_ver:
+        fpath = os.path.join(os.path.dirname(fpath), ver_file)
+        if os.path.isfile(fpath):
+            os.remove(fpath)
+        shutil.copy2(ver_template, fpath)
+
+
 def _download_frontend(pkg_path: str):
     """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
     directory."""

.github/actions/pkg-check/action.yml

@@ -50,7 +50,8 @@ runs:
     run: |
       import os, glob, pathlib, shutil
       # list folders without ending .egg-info
-      dirs = [d for d in glob.glob(os.path.join("*", "src", "*")) if not d.endswith(".egg-info")]
+      ls = glob.glob(os.path.join("*", "src", "*"))
+      dirs = [d for d in ls if os.path.isdir(d) and not d.endswith(".egg-info")]
       print(dirs)
       assert len(dirs) == ${{ inputs.nb-dirs }}
       # cleaning

.github/workflows/docs-checks.yml

@@ -21,6 +21,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
   cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
 
+env:
+  FREEZE_REQUIREMENTS: "1"
+
 defaults:
   run:
     shell: bash

.gitignore

@@ -49,6 +49,7 @@ wheels/
 .installed.cfg
 *.egg
 src/lightning/*/
+src/*/version.info
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

MANIFEST.in

@@ -3,4 +3,5 @@ exclude requirements.txt
 exclude __pycache__
 include .actions/setup_tools.py
 include .actions/assistant.py
+include src/version.info
 include *.cff  # citation info

dockers/nvidia/Dockerfile

@@ -42,9 +42,10 @@ RUN \
     cp -r lightning/*examples . && \
 
 # Installations \
-    pip install -q fire && \
     pip install "Pillow>=8.2, !=8.3.0" "cryptography>=3.4" "py>=1.10" --no-cache-dir && \
-    pip install ./lightning["extra","loggers","strategies","examples"] --no-cache-dir && \
+    # remove colossalai from requirements since they are installed separately
+    python -c "fname = 'lightning/requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)" ; \
+    PACKAGE_NAME=pytorch pip install './lightning[extra,loggers,strategies]' --no-cache-dir && \
     rm -rf lightning && \
     pip list
 
@@ -60,6 +61,6 @@ RUN \
     pip --version && \
     pip list | grep torch && \
     python -c "import torch; assert torch.__version__.startswith('$TORCH_VERSION'), torch.__version__" && \
-    python -c "import lightning as L; print(L.__version__)"
+    python -c "import pytorch_lightning as pl; print(pl.__version__)"
 
 CMD ["jupyter", "notebook", "--port=8888", "--no-browser", "--ip=0.0.0.0", "--allow-root"]

dockers/release/Dockerfile

@@ -39,13 +39,15 @@ RUN \
     fi && \
     # otherwise there is collision with folder name ans pkg name on Pypi
     cd lightning && \
-    pip install .["extra","loggers","strategies"] --no-cache-dir --find-links https://release.colossalai.org && \
+    # remove colossalai from requirements since they are installed separately
+    python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)" ; \
+    PACKAGE_NAME=pytorch pip install '.[extra,loggers,strategies]' --no-cache-dir && \
     cd .. && \
     rm -rf lightning
 
 RUN python --version && \
     pip --version && \
     pip list && \
-    python -c "import lightning as L; print(L.__version__)"
+    python -c "import pytorch_lightning as pl; print(pl.__version__)"
 
 # CMD ["/bin/bash"]

setup.py

@@ -30,8 +30,8 @@
 
 3. Building packages as sdist or binary wheel and installing or publish to PyPI afterwords you use command
     `python setup.py sdist` or `python setup.py bdist_wheel` accordingly.
-   In case you want to build just a particular package you would use exporting env. variable as above:
-   `export PACKAGE_NAME=pytorch|app|lite ; python setup.py sdist bdist_wheel`
+   In case you want to build just a particular package you want to set an environment variable:
+   `PACKAGE_NAME=lightning|pytorch|app|lite python setup.py sdist|bdist_wheel`
 
 4. Automated releasing with GitHub action is natural extension of 3) is composed of three consecutive steps:
     a) determine which packages shall be released based on version increment in `__version__.py` and eventually
@@ -73,6 +73,10 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     setup_tools = _load_py_module(name="setup_tools", location=os.path.join(_PATH_ROOT, ".actions", "setup_tools.py"))
     assistant = _load_py_module(name="assistant", location=os.path.join(_PATH_ROOT, ".actions", "assistant.py"))
 
+    if os.path.exists(_PATH_SRC):  # not a wheel install
+        # copy the version information to all packages
+        setup_tools.distribute_version(_PATH_SRC)
+
     package_to_install = _PACKAGE_NAME or "lightning"
     print(f"Installing the {package_to_install} package")  # requires `-v` to appear
     if package_to_install == "lightning":  # install everything

src/lightning/__setup__.py

@@ -35,6 +35,7 @@ def _adjust_manifest(**kwargs: Any) -> None:
         "recursive-include requirements *.txt",
         "recursive-include src/lightning/app/ui *",
         "recursive-include src/lightning/cli/*-template *",  # Add templates as build-in
+        "include src/lightning/version.info" + os.linesep,
         "include src/lightning/app/components/serve/catimage.png" + os.linesep,
         # fixme: this is strange, this shall work with setup find package - include
         "prune src/lightning_app",
@@ -60,7 +61,7 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]:
 
     return dict(
         name="lightning",
-        version=_version.version,  # todo: consider adding branch for installation from source
+        version=_version.version,
         description=_about.__docs__,
         author=_about.__author__,
         author_email=_about.__author_email__,

src/lightning/__version__.py

@@ -1 +1,9 @@
-version = "1.9.0dev"
+import os
+
+_PACKAGE_ROOT = os.path.dirname(__file__)
+_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info")
+if not os.path.exists(_SDIST_PATH):
+    # relevant for `bdist_wheel`
+    _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info")
+with open(_VERSION_PATH, encoding="utf-8") as fo:
+    version = fo.readlines()[0].strip()

src/lightning_app/__setup__.py

@@ -53,6 +53,7 @@ def _adjust_manifest(**__: Any) -> None:
         "include src/lightning_app/components/serve/catimage.png" + os.linesep,
         "recursive-include requirements/app *.txt" + os.linesep,
         "recursive-include src/lightning_app/cli/*-template *" + os.linesep,  # Add templates
+        "include src/lightning_app/version.info" + os.linesep,
     ]
 
     # TODO: remove this once lightning-ui package is ready as a dependency
@@ -76,7 +77,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]:
 
     return dict(
         name="lightning-app",
-        version=_version.version,  # todo: consider using date version + branch for installation from source
+        version=_version.version,
         description=_about.__docs__,
         author=_about.__author__,
         author_email=_about.__author_email__,

src/lightning_app/__version__.py

@@ -1 +1,9 @@
-version = "1.9.0dev"
+import os
+
+_PACKAGE_ROOT = os.path.dirname(__file__)
+_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info")
+if not os.path.exists(_SDIST_PATH):
+    # relevant for `bdist_wheel`
+    _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info")
+with open(_VERSION_PATH, encoding="utf-8") as fo:
+    version = fo.readlines()[0].strip()

src/lightning_lite/__setup__.py

@@ -52,6 +52,7 @@ def _adjust_manifest(**__: Any) -> None:
         "recursive-exclude requirements *.txt" + os.linesep,
         "recursive-include requirements/lite *.txt" + os.linesep,
         "recursive-include src/lightning_lite *.md" + os.linesep,
+        "include src/lightning_lite/version.info" + os.linesep,
     ]
 
     # TODO: remove this once lightning-ui package is ready as a dependency
@@ -72,7 +73,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]:
 
     return dict(
         name="lightning-lite",
-        version=_version.version,  # todo: consider using date version + branch for installation from source
+        version=_version.version,
         description=_about.__docs__,
         author=_about.__author__,
         author_email=_about.__author_email__,

src/lightning_lite/__version__.py

@@ -1 +1,9 @@
-version = "1.9.0dev"
+import os
+
+_PACKAGE_ROOT = os.path.dirname(__file__)
+_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info")
+if not os.path.exists(_SDIST_PATH):
+    # relevant for `bdist_wheel`
+    _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info")
+with open(_VERSION_PATH, encoding="utf-8") as fo:
+    version = fo.readlines()[0].strip()

src/pytorch_lightning/__setup__.py

@@ -54,8 +54,10 @@ def _adjust_manifest(**__: Any) -> None:
         "recursive-exclude requirements *.txt" + os.linesep,
         "recursive-include requirements/lite *.txt" + os.linesep,
         "recursive-include src/lightning_lite *.md" + os.linesep,
+        "include src/lightning_lite/version.info" + os.linesep,
         "recursive-include src/pytorch_lightning *.md" + os.linesep,
         "recursive-include requirements/pytorch *.txt" + os.linesep,
+        "include src/pytorch_lightning/version.info" + os.linesep,
         "include src/pytorch_lightning/py.typed" + os.linesep,  # marker file for PEP 561
     ]
     with open(manifest_path, "w") as fp:
@@ -72,7 +74,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]:
     )
     return dict(
         name="pytorch-lightning",
-        version=_version.version,  # todo: consider using date version + branch for installation from source
+        version=_version.version,
         description=_about.__docs__,
         author=_about.__author__,
         author_email=_about.__author_email__,

src/pytorch_lightning/__version__.py

@@ -1 +1,9 @@
-version = "1.9.0dev"
+import os
+
+_PACKAGE_ROOT = os.path.dirname(__file__)
+_SDIST_PATH = _VERSION_PATH = os.path.join(os.path.dirname(_PACKAGE_ROOT), "version.info")
+if not os.path.exists(_SDIST_PATH):
+    # relevant for `bdist_wheel`
+    _VERSION_PATH = os.path.join(_PACKAGE_ROOT, "version.info")
+with open(_VERSION_PATH, encoding="utf-8") as fo:
+    version = fo.readlines()[0].strip()

src/version.info

@@ -0,0 +1 @@
+1.9.0dev