From 3bd55a6057691930850c6805aa4e62a6b9e2c08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 4 Nov 2022 17:51:03 +0100 Subject: [PATCH] Do not modify PACKAGE_NAME on install (#15493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Do not modify PACKAGE_NAME on install * Fix ci pkg action * Required * Typos * Apply suggestions from code review * Undo defaults * Cleanup * Implement idea * Fuck * Apps mock fix * Fix app-pytest with PKG_NAME=app * Justus suggestion * Debug Windows * Update setup.py Co-authored-by: Adrian Wälchli * Revert "Debug Windows" This reverts commit 9fe3ba366545ba9e308b0757876279b1fff8a553. * SSH action * Crazy bug * Revert "SSH action" This reverts commit 5061e8e7d6fdf47b38c676623b02387cd451d403. * Package import step * Avoid env conflict * Debug * Whitespace * Try removing existing lite build * This should be redundant now * Add back env now that source-lit is gone * Remove download artifact * checkgroup * TODOs suggested by Jirka * _ * Revert "_". These are local variables, do not need protected This reverts commit 8340b85991bb6927d851d86861b7efe2c845377b. Co-authored-by: Jirka Borovec Co-authored-by: Adrian Wälchli (cherry picked from commit f392180c385e211914ec973800882534662e7fd6) --- .actions/setup_tools.py | 22 +----- .azure/gpu-benchmark.yml | 10 --- .azure/gpu-tests-pytorch.yml | 10 --- .azure/hpu-tests.yml | 10 --- .azure/ipu-tests.yml | 10 --- .github/actions/pkg-check/action.yml | 4 +- .github/actions/pkg-install/action.yml | 27 +++---- .github/checkgroup.yml | 12 +-- .github/workflows/ci-app-examples.yml | 3 +- .github/workflows/ci-app-tests.yml | 3 +- .github/workflows/ci-pkg-install.yml | 58 ++------------- .github/workflows/ci-pytorch-tests-slow.yml | 9 --- .github/workflows/ci-pytorch-tests.yml | 10 --- .github/workflows/docs-checks.yml | 37 ++-------- requirements/pytorch/base.txt | 1 - setup.py | 73 +++++++++---------- src/lightning_app/testing/testing.py | 11 ++- src/lightning_app/utilities/cli_helpers.py | 3 +- src/pytorch_lightning/__setup__.py | 12 +-- tests/tests_app/cli/test_cli.py | 12 +-- tests/tests_app/cli/test_cmd_react_ui_init.py | 2 +- 21 files changed, 91 insertions(+), 248 deletions(-) diff --git a/.actions/setup_tools.py b/.actions/setup_tools.py index a8c00e797598d..77034a9b26580 100644 --- a/.actions/setup_tools.py +++ b/.actions/setup_tools.py @@ -232,8 +232,10 @@ def create_mirror_package(src_folder: str, lit_pkg_mapping: dict) -> None: ... {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"} ... ) """ - for lit_name, pkg_name in lit_pkg_mapping.items(): - copy_adjusted_modules(src_folder, pkg_name, lit_name, lit_pkg_mapping) + mapping = lit_pkg_mapping.copy() + mapping.pop("lightning", None) # pop this key to avoid replacing `lightning` to `lightning.lightning` + for lit_name, pkg_name in mapping.items(): + copy_adjusted_modules(src_folder, pkg_name, lit_name, mapping) def _download_frontend(pkg_path: str): @@ -272,19 +274,3 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme requires = list(chain(*requires)) with open(os.path.join(req_dir, "base.txt"), "w") as fp: fp.writelines([ln + os.linesep for ln in requires]) - - -def set_actual_version_from_src(req_path: str, src_root: str, pkg_name: str) -> None: - """Setting actual version from source code for a given package.""" - with open(req_path, encoding="utf-8") as fo: - lines = fo.readlines() - ver = parse_version_from_file(os.path.join(src_root, pkg_name.replace("-", "_"))) - for i, ln in enumerate(lines): - reqs = list(parse_requirements([ln])) - if not reqs: - continue - if reqs[0].name == pkg_name: - lines[i] = f"{pkg_name}=={ver}{os.linesep}" - - with open(req_path, "w", encoding="utf-8") as fw: - fw.writelines(lines) diff --git a/.azure/gpu-benchmark.yml b/.azure/gpu-benchmark.yml index d8faf43e5d17a..8fad0d69c15d1 100644 --- a/.azure/gpu-benchmark.yml +++ b/.azure/gpu-benchmark.yml @@ -51,16 +51,6 @@ jobs: echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html" displayName: 'set env. vars' - - bash: | - pip install -e . --find-links ${TORCH_URL} - git checkout -- setup.py MANIFEST.in - env: - PACKAGE_NAME: lite - FREEZE_REQUIREMENTS: 1 - # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi - # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache - displayName: 'Install Lite pkg' - - bash: | pip install -e .[strategies] --find-links ${TORCH_URL} pip list diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index c3cf3849a5c5b..05571269a99a7 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -93,16 +93,6 @@ jobs: python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${PYTORCH_VERSION} displayName: 'Adjust dependencies' - - bash: | - pip install -e . --find-links ${TORCH_URL} - git checkout -- setup.py MANIFEST.in - env: - PACKAGE_NAME: "lite" - FREEZE_REQUIREMENTS: "1" - # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi - # also installing from source not from on-the-fly created package to prevent accidental interaction with cache - displayName: 'Install Lite pkg' - - bash: pip install -e .[strategies] -r requirements/pytorch/devel.txt -r requirements/pytorch/examples.txt --find-links ${TORCH_URL} env: PACKAGE_NAME: "pytorch" diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml index c29547911a462..8281f9e5c5fa6 100644 --- a/.azure/hpu-tests.yml +++ b/.azure/hpu-tests.yml @@ -58,16 +58,6 @@ jobs: sudo pip install pip -U displayName: 'Instance HW info' - - bash: | - pip install -e . - git checkout -- setup.py MANIFEST.in - env: - PACKAGE_NAME: "lite" - FREEZE_REQUIREMENTS: "1" - # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi - # also installing from source not from on-the-fly created package to prevent accidental interaction with cache - displayName: 'Install Lite pkg' - - bash: | set -e pip --version diff --git a/.azure/ipu-tests.yml b/.azure/ipu-tests.yml index 690b1acc32b6d..972bf1e95a06b 100644 --- a/.azure/ipu-tests.yml +++ b/.azure/ipu-tests.yml @@ -62,16 +62,6 @@ jobs: done displayName: "Reset IPU devices" - - bash: | - pip install -e . - git checkout -- setup.py MANIFEST.in - env: - PACKAGE_NAME: "lite" - FREEZE_REQUIREMENTS: "1" - # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi - # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache - displayName: 'Install Lite pkg' - - bash: | python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml index 9f86319774585..6680f945d589d 100644 --- a/.github/actions/pkg-check/action.yml +++ b/.github/actions/pkg-check/action.yml @@ -39,13 +39,13 @@ runs: shell: bash - name: Unzip packages - if: ${{ inputs.pkg-name != '' }} + if: ${{ inputs.pkg-name != 'lightning' }} working-directory: dist run: for file in `ls *.gz`; do tar -xzf $file; done shell: bash - name: Check single pkg/folder - if: ${{ inputs.pkg-name != '' }} + if: ${{ inputs.pkg-name != 'lightning' }} working-directory: dist run: | import os, glob, pathlib, shutil diff --git a/.github/actions/pkg-install/action.yml b/.github/actions/pkg-install/action.yml index 155c4edf99997..0e0751c217a95 100644 --- a/.github/actions/pkg-install/action.yml +++ b/.github/actions/pkg-install/action.yml @@ -1,28 +1,21 @@ -name: Install and check package -description: installing and validationg the package +name: Install and validate the package +description: Install and validate the package inputs: pkg-name: - description: package name for import - required: false - default: "" + description: Package name to import + required: true pip-flags: - description: additional pil install flags + description: Additional pip install flags required: false default: "" runs: using: "composite" steps: - - name: Determine package name - if: ${{ inputs.pkg-import == '' }} - working-directory: ./dist - run: python -c "import glob ; ls = glob.glob('*.tar.gz') ; name = '_'.join(ls[0].split('-')[:-1]) ; print(f'PKG_NAME={name}')" >> $GITHUB_ENV - shell: bash - - - name: Pass package name - if: ${{ inputs.pkg-import != '' }} - run: echo "PKG_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV + - name: Choose package import + run: | + python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV shell: bash - name: Install package - archive @@ -30,7 +23,7 @@ runs: run: | pip install *.tar.gz ${{ inputs.pip-flags }} pip list | grep lightning - python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)" + python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)" shell: bash - name: Install package - wheel @@ -38,5 +31,5 @@ runs: run: | pip install *.whl ${{ inputs.pip-flags }} pip list | grep lightning - python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)" + python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)" shell: bash diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 86d4dc9a0d07d..fce40e1c23e2d 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -358,21 +358,21 @@ subprojects: - "install-pkg (ubuntu-22.04, lite, 3.10)" - "install-pkg (ubuntu-22.04, pytorch, 3.7)" - "install-pkg (ubuntu-22.04, pytorch, 3.10)" - - "install-pkg (ubuntu-22.04, 3.7)" - - "install-pkg (ubuntu-22.04, 3.10)" + - "install-pkg (ubuntu-22.04, lightning, 3.7)" + - "install-pkg (ubuntu-22.04, lightning, 3.10)" - "install-pkg (macOS-12, app, 3.7)" - "install-pkg (macOS-12, app, 3.10)" - "install-pkg (macOS-12, lite, 3.7)" - "install-pkg (macOS-12, lite, 3.10)" - "install-pkg (macOS-12, pytorch, 3.7)" - "install-pkg (macOS-12, pytorch, 3.10)" - - "install-pkg (macOS-12, 3.7)" - - "install-pkg (macOS-12, 3.10)" + - "install-pkg (macOS-12, lightning, 3.7)" + - "install-pkg (macOS-12, lightning, 3.10)" - "install-pkg (windows-2022, app, 3.7)" - "install-pkg (windows-2022, app, 3.10)" - "install-pkg (windows-2022, lite, 3.7)" - "install-pkg (windows-2022, lite, 3.10)" - "install-pkg (windows-2022, pytorch, 3.7)" - "install-pkg (windows-2022, pytorch, 3.10)" - - "install-pkg (windows-2022, 3.7)" - - "install-pkg (windows-2022, 3.10)" + - "install-pkg (windows-2022, lightning, 3.7)" + - "install-pkg (windows-2022, lightning, 3.10)" diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml index 66d321179aed6..2f4447c59faf4 100644 --- a/.github/workflows/ci-app-examples.yml +++ b/.github/workflows/ci-app-examples.yml @@ -99,8 +99,7 @@ jobs: - name: Adjust examples if: ${{ matrix.pkg-name != 'lightning' }} run: | - python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app" - python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app" + python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app,lightning" --target_import="lightning_app,lightning_app" - name: Switch coverage scope run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml index 38e337a3f392f..3a7a56308a7bd 100644 --- a/.github/workflows/ci-app-tests.yml +++ b/.github/workflows/ci-app-tests.yml @@ -104,8 +104,7 @@ jobs: - name: Adjust examples if: ${{ matrix.pkg-name != 'lightning' }} run: | - python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app" - python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app" + python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app,lightning" --target_import="lightning_app,lightning_app" - name: Switch coverage scope run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index 6c05180036a8f..b1fad271779fc 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -25,56 +25,17 @@ defaults: jobs: - init-pypi: - runs-on: ubuntu-20.04 - steps: - - run: mkdir pypi && touch pypi/.placeholder - - uses: actions/upload-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi - - build-pypi: - if: github.event.pull_request.draft == false - needs: init-pypi - # This serves to create packages for potential internal dependencies - runs-on: ubuntu-20.04 - strategy: - fail-fast: true - max-parallel: 1 - matrix: - pkg-name: ["lite"] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - uses: actions/download-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi - - name: Build package - env: - PACKAGE_NAME: ${{ matrix.pkg-name }} - run: | - python setup.py sdist - cp dist/* pypi/ - - uses: actions/upload-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi - install-pkg: - needs: build-pypi runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-22.04, macOS-12, windows-2022] - pkg-name: ["app", "lite", "pytorch", ""] + pkg-name: ["app", "lite", "pytorch", "lightning"] python-version: ["3.7" , "3.10"] steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -86,27 +47,22 @@ jobs: python -m pytest setup_tools.py - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg-name }}' == 'pytorch' else 1))" >> $GITHUB_ENV + - uses: ./.github/actions/pkg-check with: pkg-name: ${{ matrix.pkg-name }} nb-dirs: ${{ env.NB_DIRS }} - - uses: actions/download-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi - - run: ls -l pypi/ - uses: ./.github/actions/pkg-install with: - pip-flags: "--pre --find-links ../pypi/" + pkg-name: ${{ matrix.pkg-name }} - name: Run CLI # todo: add testing for `lightning_app` - if: ${{ matrix.pkg-name == '' }} + if: ${{ matrix.pkg-name == 'lightning' }} run: python -m lightning --version - name: DocTest package run: | - pip list - scope=$(python -c "lut = {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning'} ; print(lut.get('${{matrix.pkg-name}}', 'lightning'))") - python -m pytest src/${scope} --ignore-glob="**/cli/*-template/**" + PKG_NAME=$(python -c "print({'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])") + python -m pytest src/${PKG_NAME} --ignore-glob="**/cli/*-template/**" diff --git a/.github/workflows/ci-pytorch-tests-slow.yml b/.github/workflows/ci-pytorch-tests-slow.yml index 053e47420551e..b6f2bd1884e74 100644 --- a/.github/workflows/ci-pytorch-tests-slow.yml +++ b/.github/workflows/ci-pytorch-tests-slow.yml @@ -56,15 +56,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - name: Install lite for PL - # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi - # also installing from source not from on-the-fly created package to prevent accidental interaction with cache - env: - PACKAGE_NAME: "lite" - run: | - pip install -e . --upgrade --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - git checkout -- setup.py MANIFEST.in - - name: Install package & dependencies env: PACKAGE_NAME: "pytorch" diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml index 030fbac027958..7ec6a896864fb 100644 --- a/.github/workflows/ci-pytorch-tests.yml +++ b/.github/workflows/ci-pytorch-tests.yml @@ -124,16 +124,6 @@ jobs: - name: Switch PyTorch URL run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV - - name: Install lite for PL - # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi - # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache - if: ${{ matrix.pkg-name != 'lightning' }} - env: - PACKAGE_NAME: "lite" - run: | - pip install -e . --upgrade --find-links ${TORCH_URL} - git checkout -- setup.py MANIFEST.in - - name: Install package & dependencies env: PACKAGE_NAME: ${{ matrix.pkg-name }} diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index a5bdd1ddbde52..35ff31364ab0f 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -26,30 +26,8 @@ defaults: shell: bash jobs: - - build-pypi: - # This serves to create packages for potential internal dependencies - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Build package - env: - PACKAGE_NAME: "lite" - run: python setup.py sdist - - uses: actions/upload-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: dist - make-doctest: runs-on: ubuntu-20.04 - # make it depending on build docs to reduce load - needs: build-pypi strategy: fail-fast: false matrix: @@ -61,10 +39,7 @@ jobs: with: submodules: true ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi + - uses: actions/setup-python@v4 with: python-version: "3.9" @@ -75,6 +50,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }} aws-region: us-east-1 + - run: aws s3 sync s3://sphinx-packages/ pypi/ if: ${{ matrix.pkg-name == 'app' }} @@ -99,7 +75,7 @@ jobs: if: ${{ matrix.pkg-name == 'app' }} run: | pip install -e . -U -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi - git checkout -- setup.py MANIFEST.in + git checkout -- MANIFEST.in - name: Adjust docs refs if: ${{ matrix.pkg-name == 'app' }} @@ -123,7 +99,6 @@ jobs: make coverage make-html: - needs: build-pypi runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -136,10 +111,7 @@ jobs: with: submodules: true ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi + - uses: actions/setup-python@v4 with: python-version: "3.9" @@ -150,6 +122,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }} aws-region: us-east-1 + - run: aws s3 sync s3://sphinx-packages/ pypi/ if: ${{ matrix.pkg-name != 'pytorch' }} diff --git a/requirements/pytorch/base.txt b/requirements/pytorch/base.txt index 765be74803fe0..e3eae1cd66ce8 100644 --- a/requirements/pytorch/base.txt +++ b/requirements/pytorch/base.txt @@ -3,7 +3,6 @@ numpy>=1.17.2, <1.23.1 torch>=1.9.*, <1.13.0 -lightning-lite # actual version comes from src tqdm>=4.57.0, <4.65.0 PyYAML>=5.4, <=6.0 fsspec[http]>2021.06.0, <2022.8.0 diff --git a/setup.py b/setup.py index b400133b14da1..ce2505223302c 100755 --- a/setup.py +++ b/setup.py @@ -38,15 +38,6 @@ compared against PyPI registry b) with a parameterization build desired packages in to standard `dist/` folder c) validate packages and publish to PyPI - - -| Installation | PIP version * | Pkg version ** | -| -------------- | ------------------- | --------------- | -| source | calendar + branch | semantic | -| PyPI | semantic | semantic | - -* shown version while calling `pip list | grep lightning` -** shown version in python `from import __version__` """ import os from importlib.util import module_from_spec, spec_from_file_location @@ -54,30 +45,21 @@ from setuptools import setup -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") -_PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"} -_REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME") +_PACKAGE_MAPPING = { + "lightning": "lightning", + "pytorch": "pytorch_lightning", + "app": "lightning_app", + "lite": "lightning_lite", +} # https://packaging.python.org/guides/single-sourcing-package-version/ # http://blog.ionelmc.ro/2014/05/25/python-packaging/ _PATH_ROOT = os.path.dirname(__file__) _PATH_SRC = os.path.join(_PATH_ROOT, "src") _PATH_REQUIRE = os.path.join(_PATH_ROOT, "requirements") -_PATH_SETUP = os.path.join(_PATH_SRC, _REAL_PKG_NAME, "__setup__.py") -if not os.path.isfile(_PATH_SETUP): - _PATH_SETUP = os.path.join(_PATH_SRC, "lightning", "__setup__.py") _FREEZE_REQUIREMENTS = bool(int(os.environ.get("FREEZE_REQUIREMENTS", 0))) -# Hardcode the env variable from time of package creation, otherwise it fails during installation -with open(__file__) as fp: - lines = fp.readlines() -for i, ln in enumerate(lines): - if ln.startswith("_PACKAGE_NAME = "): - lines[i] = f'_PACKAGE_NAME = "{_PACKAGE_NAME}"{os.linesep}' -with open(__file__, "w") as fp: - fp.writelines(lines) - - def _load_py_module(name: str, location: str) -> ModuleType: spec = spec_from_file_location(name, location) assert spec, f"Failed to load module {name} from {location}" @@ -87,19 +69,30 @@ def _load_py_module(name: str, location: str) -> ModuleType: return py -# https://packaging.python.org/discussions/install-requires-vs-requirements / -# keep the meta-data here for simplicity in reading this file... it's not obvious -# what happens and to non-engineers they won't know to look in init ... -# the goal of the project is simplicity for researchers, don't want to add too much -# engineer specific practices if __name__ == "__main__": - _SETUP_TOOLS = _load_py_module(name="setup_tools", location=os.path.join(".actions", "setup_tools.py")) - - if _PACKAGE_NAME not in _PACKAGE_MAPPING: # install everything - _SETUP_TOOLS._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS) - - _SETUP_TOOLS.create_mirror_package(os.path.join(_PATH_ROOT, "src"), _PACKAGE_MAPPING) - - _SETUP_MODULE = _load_py_module(name="pkg_setup", location=_PATH_SETUP) - _SETUP_MODULE._adjust_manifest(pkg_name=_REAL_PKG_NAME) - setup(**_SETUP_MODULE._setup_args(pkg_name=_REAL_PKG_NAME)) + setup_tools = _load_py_module(name="setup_tools", location=os.path.join(".actions", "setup_tools.py")) + + package_to_install = _PACKAGE_NAME or "lightning" + print(f"Installing the {package_to_install} package") # requires `-v` to appear + if package_to_install == "lightning": + # install everything + setup_tools._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS) + setup_tools.create_mirror_package(_PATH_SRC, _PACKAGE_MAPPING) + elif package_to_install not in _PACKAGE_MAPPING: + raise ValueError(f"Unexpected package name: {_PACKAGE_NAME}. Possible choices are: {list(_PACKAGE_MAPPING)}") + + # if `_PACKAGE_NAME` is not set, iterate over all possible packages until we find one that can be installed. + # this is useful for installing existing wheels, as the user wouldn't set this environment variable, but the wheel + # should have included only the relevant files of the package to install + possible_packages = _PACKAGE_MAPPING.values() if _PACKAGE_NAME is None else [_PACKAGE_MAPPING[_PACKAGE_NAME]] + for pkg in possible_packages: + pkg_setup = os.path.join(_PATH_SRC, pkg, "__setup__.py") + if os.path.exists(pkg_setup): + print(f"{pkg_setup} exists. Running `setuptools.setup`") + setup_module = _load_py_module(name=f"{pkg}_setup", location=pkg_setup) + setup_module._adjust_manifest(pkg_name=pkg) + setup_args = setup_module._setup_args(pkg_name=pkg) + setup(**setup_args) + break + else: + raise RuntimeError(f"Something's wrong, no package was installed. Package name: {_PACKAGE_NAME}") diff --git a/src/lightning_app/testing/testing.py b/src/lightning_app/testing/testing.py index a7a2b70d474a6..6bf17707cf512 100644 --- a/src/lightning_app/testing/testing.py +++ b/src/lightning_app/testing/testing.py @@ -7,7 +7,7 @@ import sys import tempfile import time -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from multiprocessing import Process from subprocess import Popen from time import sleep @@ -137,9 +137,12 @@ def application_testing( from click.testing import CliRunner - with mock.patch("lightning.LightningApp", lightning_app_cls), mock.patch( - "lightning_app.LightningApp", lightning_app_cls - ): + patch1 = mock.patch("lightning_app.LightningApp", lightning_app_cls) + # we need to patch both only with the mirror package + patch2 = ( + mock.patch("lightning.LightningApp", lightning_app_cls) if "lightning.app" in sys.modules else nullcontext() + ) + with patch1, patch2: original = sys.argv sys.argv = command_line runner = CliRunner() diff --git a/src/lightning_app/utilities/cli_helpers.py b/src/lightning_app/utilities/cli_helpers.py index 2a2bdda831141..b3b59aa106074 100644 --- a/src/lightning_app/utilities/cli_helpers.py +++ b/src/lightning_app/utilities/cli_helpers.py @@ -301,7 +301,8 @@ def _check_environment_and_redirect(): """ env_executable = shutil.which("python") - if env_executable != sys.executable: + # on windows, the extension might be different, where one uses `.EXE` and the other `.exe` + if env_executable.lower() != sys.executable.lower(): logger.info( "Lightning is running from outside your current environment. Switching to your current environment." ) diff --git a/src/pytorch_lightning/__setup__.py b/src/pytorch_lightning/__setup__.py index 205aefe48cc00..442bda630b884 100644 --- a/src/pytorch_lightning/__setup__.py +++ b/src/pytorch_lightning/__setup__.py @@ -52,6 +52,8 @@ def _adjust_manifest(**__: Any) -> None: lines += [ "recursive-exclude src *.md" + os.linesep, "recursive-exclude requirements *.txt" + os.linesep, + "recursive-include requirements/lite *.txt" + os.linesep, + "recursive-include src/lightning_lite *.md" + os.linesep, "recursive-include src/pytorch_lightning *.md" + os.linesep, "recursive-include requirements/pytorch *.txt" + os.linesep, "include src/pytorch_lightning/py.typed" + os.linesep, # marker file for PEP 561 @@ -63,10 +65,6 @@ def _adjust_manifest(**__: Any) -> None: def _setup_args(**__: Any) -> Dict[str, Any]: _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py") _setup_tools = _load_py_module("setup_tools", _path_setup_tools) - if os.path.isdir(os.path.join(_SOURCE_ROOT, "lightning_lite")): - _setup_tools.set_actual_version_from_src( - os.path.join(_PATH_REQUIREMENTS, "base.txt"), _SOURCE_ROOT, "lightning-lite" - ) _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py")) _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py")) _long_description = _setup_tools.load_readme_description( @@ -86,8 +84,8 @@ def _setup_args(**__: Any) -> Dict[str, Any]: include=[ "pytorch_lightning", "pytorch_lightning.*", - "lightning_lite", # TODO: remove after the first standalone Lite release - "lightning_lite.*", # TODO: remove after the first standalone Lite release + "lightning_lite", + "lightning_lite.*", ], ), package_dir={"": "src"}, @@ -98,6 +96,8 @@ def _setup_args(**__: Any) -> Dict[str, Any]: keywords=["deep learning", "pytorch", "AI"], python_requires=">=3.7", setup_requires=[], + # TODO: aggregate pytorch and lite requirements as we include its source code directly in this package. + # this is not a problem yet because lite's base requirements are all included in pytorch's base requirements install_requires=_setup_tools.load_requirements( _PATH_REQUIREMENTS, unfreeze="" if _FREEZE_REQUIREMENTS else "all" ), diff --git a/tests/tests_app/cli/test_cli.py b/tests/tests_app/cli/test_cli.py index 9ccf543f9dac7..620bd6ea9e74f 100644 --- a/tests/tests_app/cli/test_cli.py +++ b/tests/tests_app/cli/test_cli.py @@ -52,7 +52,7 @@ def test_commands(command): def test_main_lightning_cli_no_arguments(): """Validate the Lightning CLI without args.""" - res = os.popen("python -m lightning").read() + res = os.popen("lightning").read() assert "login " in res assert "logout " in res assert "run " in res @@ -67,7 +67,7 @@ def test_main_lightning_cli_no_arguments(): def test_main_lightning_cli_help(): """Validate the Lightning CLI.""" - res = os.popen("python -m lightning --help").read() + res = os.popen("lightning --help").read() assert "login " in res assert "logout " in res assert "run " in res @@ -79,7 +79,7 @@ def test_main_lightning_cli_help(): assert "add " in res assert "remove " in res - res = os.popen("python -m lightning run --help").read() + res = os.popen("lightning run --help").read() assert "app " in res # hidden run commands should not appear in the help text @@ -89,12 +89,12 @@ def test_main_lightning_cli_help(): assert "frontend" not in res # inspect show group - res = os.popen("python -m lightning show --help").read() + res = os.popen("lightning show --help").read() assert "logs " in res assert "cluster " in res # inspect show cluster group - res = os.popen("python -m lightning show cluster --help").read() + res = os.popen("lightning show cluster --help").read() assert "logs " in res @@ -187,7 +187,7 @@ def test_cli_logout(exists: mock.MagicMock, unlink: mock.MagicMock, creds: bool) def test_lightning_cli_version(): - res = os.popen("python -m lightning --version").read() + res = os.popen("lightning --version").read() assert __version__ in res diff --git a/tests/tests_app/cli/test_cmd_react_ui_init.py b/tests/tests_app/cli/test_cmd_react_ui_init.py index 6b3bd530a6ccd..0cc84525bb906 100644 --- a/tests/tests_app/cli/test_cmd_react_ui_init.py +++ b/tests/tests_app/cli/test_cmd_react_ui_init.py @@ -32,7 +32,7 @@ def test_missing_yarn(): @_RunIf(skip_windows=True) def test_copy_and_setup_react_ui(tmpdir): dest_dir = os.path.join(tmpdir, "react-ui") - subprocess.Popen(["python", "-m", "lightning", "init", "react-ui", "--dest_dir", dest_dir]).wait() + subprocess.Popen(["lightning", "init", "react-ui", "--dest_dir", dest_dir]).wait() # make sure package is minimal files = sorted(f for f in os.listdir(dest_dir) if f != "__pycache__")