Do not modify PACKAGE_NAME on install (#15493)

* Do not modify PACKAGE_NAME on install * Fix ci pkg action * Required * Typos * Apply suggestions from code review * Undo defaults * Cleanup * Implement idea * Fuck * Apps mock fix * Fix app-pytest with PKG_NAME=app * Justus suggestion * Debug Windows * Update setup.py Co-authored-by: Adrian Wälchli <[email protected]> * Revert "Debug Windows" This reverts commit 9fe3ba3. * SSH action * Crazy bug * Revert "SSH action" This reverts commit 5061e8e. * Package import step * Avoid env conflict * Debug * Whitespace * Try removing existing lite build * This should be redundant now * Add back env now that source-lit is gone * Remove download artifact * checkgroup * TODOs suggested by Jirka * _ * Revert "_". These are local variables, do not need protected This reverts commit 8340b85. Co-authored-by: Jirka Borovec <[email protected]> Co-authored-by: Adrian Wälchli <[email protected]> (cherry picked from commit f392180)
Lightning-AI · Nov 10, 2022 · 3bd55a6 · 3bd55a6
1 parent 53f7f0c
commit 3bd55a6
Show file tree

Hide file tree

Showing 21 changed files with 91 additions and 248 deletions.
diff --git a/.actions/setup_tools.py b/.actions/setup_tools.py
@@ -232,8 +232,10 @@ def create_mirror_package(src_folder: str, lit_pkg_mapping: dict) -> None:
     ...     {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"}
     ... )
     """
-    for lit_name, pkg_name in lit_pkg_mapping.items():
-        copy_adjusted_modules(src_folder, pkg_name, lit_name, lit_pkg_mapping)
+    mapping = lit_pkg_mapping.copy()
+    mapping.pop("lightning", None)  # pop this key to avoid replacing `lightning` to `lightning.lightning`
+    for lit_name, pkg_name in mapping.items():
+        copy_adjusted_modules(src_folder, pkg_name, lit_name, mapping)
 
 
 def _download_frontend(pkg_path: str):
@@ -272,19 +274,3 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
     requires = list(chain(*requires))
     with open(os.path.join(req_dir, "base.txt"), "w") as fp:
         fp.writelines([ln + os.linesep for ln in requires])
-
-
-def set_actual_version_from_src(req_path: str, src_root: str, pkg_name: str) -> None:
-    """Setting actual version from source code for a given package."""
-    with open(req_path, encoding="utf-8") as fo:
-        lines = fo.readlines()
-    ver = parse_version_from_file(os.path.join(src_root, pkg_name.replace("-", "_")))
-    for i, ln in enumerate(lines):
-        reqs = list(parse_requirements([ln]))
-        if not reqs:
-            continue
-        if reqs[0].name == pkg_name:
-            lines[i] = f"{pkg_name}=={ver}{os.linesep}"
-
-    with open(req_path, "w", encoding="utf-8") as fw:
-        fw.writelines(lines)
diff --git a/.azure/gpu-benchmark.yml b/.azure/gpu-benchmark.yml
@@ -51,16 +51,6 @@ jobs:
         echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
       displayName: 'set env. vars'
 
-    - bash: |
-        pip install -e . --find-links ${TORCH_URL}
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: lite
-        FREEZE_REQUIREMENTS: 1
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: |
         pip install -e .[strategies] --find-links ${TORCH_URL}
         pip list

diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml
@@ -93,16 +93,6 @@ jobs:
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${PYTORCH_VERSION}
       displayName: 'Adjust dependencies'
 
-    - bash: |
-        pip install -e . --find-links ${TORCH_URL}
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: "lite"
-        FREEZE_REQUIREMENTS: "1"
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: pip install -e .[strategies] -r requirements/pytorch/devel.txt -r requirements/pytorch/examples.txt --find-links ${TORCH_URL}
       env:
         PACKAGE_NAME: "pytorch"

diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml
@@ -58,16 +58,6 @@ jobs:
         sudo pip install pip -U
       displayName: 'Instance HW info'
 
-    - bash: |
-        pip install -e .
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: "lite"
-        FREEZE_REQUIREMENTS: "1"
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: |
         set -e
         pip --version

diff --git a/.azure/ipu-tests.yml b/.azure/ipu-tests.yml
@@ -62,16 +62,6 @@ jobs:
         done
       displayName: "Reset IPU devices"
 
-    - bash: |
-        pip install -e .
-        git checkout -- setup.py MANIFEST.in
-      env:
-        PACKAGE_NAME: "lite"
-        FREEZE_REQUIREMENTS: "1"
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
-      displayName: 'Install Lite pkg'
-
     - bash: |
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt

diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml
@@ -39,13 +39,13 @@ runs:
     shell: bash
 
   - name: Unzip packages
-    if: ${{ inputs.pkg-name != '' }}
+    if: ${{ inputs.pkg-name != 'lightning' }}
     working-directory: dist
     run: for file in `ls *.gz`; do tar -xzf $file; done
     shell: bash
 
   - name: Check single pkg/folder
-    if: ${{ inputs.pkg-name != '' }}
+    if: ${{ inputs.pkg-name != 'lightning' }}
     working-directory: dist
     run: |
       import os, glob, pathlib, shutil

diff --git a/.github/actions/pkg-install/action.yml b/.github/actions/pkg-install/action.yml
@@ -1,42 +1,35 @@
-name: Install and check package
-description: installing and validationg the package
+name: Install and validate the package
+description: Install and validate the package
 
 inputs:
   pkg-name:
-    description: package name for import
-    required: false
-    default: ""
+    description: Package name to import
+    required: true
   pip-flags:
-    description: additional pil install flags
+    description: Additional pip install flags
     required: false
     default: ""
 
 runs:
   using: "composite"
   steps:
-  - name: Determine package name
-    if: ${{ inputs.pkg-import == '' }}
-    working-directory: ./dist
-    run: python -c "import glob ; ls = glob.glob('*.tar.gz') ; name = '_'.join(ls[0].split('-')[:-1]) ; print(f'PKG_NAME={name}')" >> $GITHUB_ENV
-    shell: bash
-
-  - name: Pass package name
-    if: ${{ inputs.pkg-import != '' }}
-    run: echo "PKG_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV
+  - name: Choose package import
+    run: |
+      python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV
     shell: bash
 
   - name: Install package - archive
     working-directory: ./dist
     run: |
       pip install *.tar.gz ${{ inputs.pip-flags }}
       pip list | grep lightning
-      python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)"
+      python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash
 
   - name: Install package - wheel
     working-directory: ./dist
     run: |
       pip install *.whl ${{ inputs.pip-flags }}
       pip list | grep lightning
-      python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)"
+      python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash
diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml
@@ -358,21 +358,21 @@ subprojects:
       - "install-pkg (ubuntu-22.04, lite, 3.10)"
       - "install-pkg (ubuntu-22.04, pytorch, 3.7)"
       - "install-pkg (ubuntu-22.04, pytorch, 3.10)"
-      - "install-pkg (ubuntu-22.04, 3.7)"
-      - "install-pkg (ubuntu-22.04, 3.10)"
+      - "install-pkg (ubuntu-22.04, lightning, 3.7)"
+      - "install-pkg (ubuntu-22.04, lightning, 3.10)"
       - "install-pkg (macOS-12, app, 3.7)"
       - "install-pkg (macOS-12, app, 3.10)"
       - "install-pkg (macOS-12, lite, 3.7)"
       - "install-pkg (macOS-12, lite, 3.10)"
       - "install-pkg (macOS-12, pytorch, 3.7)"
       - "install-pkg (macOS-12, pytorch, 3.10)"
-      - "install-pkg (macOS-12, 3.7)"
-      - "install-pkg (macOS-12, 3.10)"
+      - "install-pkg (macOS-12, lightning, 3.7)"
+      - "install-pkg (macOS-12, lightning, 3.10)"
       - "install-pkg (windows-2022, app, 3.7)"
       - "install-pkg (windows-2022, app, 3.10)"
       - "install-pkg (windows-2022, lite, 3.7)"
       - "install-pkg (windows-2022, lite, 3.10)"
       - "install-pkg (windows-2022, pytorch, 3.7)"
       - "install-pkg (windows-2022, pytorch, 3.10)"
-      - "install-pkg (windows-2022, 3.7)"
-      - "install-pkg (windows-2022, 3.10)"
+      - "install-pkg (windows-2022, lightning, 3.7)"
+      - "install-pkg (windows-2022, lightning, 3.10)"
diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml
@@ -99,8 +99,7 @@ jobs:
       - name: Adjust examples
         if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app"
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app"
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app,lightning" --target_import="lightning_app,lightning_app"
 
       - name: Switch coverage scope
         run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV

diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml
@@ -104,8 +104,7 @@ jobs:
       - name: Adjust examples
         if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app"
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app"
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app,lightning" --target_import="lightning_app,lightning_app"
 
       - name: Switch coverage scope
         run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV

diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml
@@ -25,56 +25,17 @@ defaults:
 
 jobs:
 
-  init-pypi:
-    runs-on: ubuntu-20.04
-    steps:
-    - run: mkdir pypi && touch pypi/.placeholder
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ci-packages-${{ github.sha }}
-        path: pypi
-
-  build-pypi:
-    if: github.event.pull_request.draft == false
-    needs: init-pypi
-    # This serves to create packages for potential internal dependencies
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: true
-      max-parallel: 1
-      matrix:
-        pkg-name: ["lite"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-      - uses: actions/download-artifact@v3
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-      - name: Build package
-        env:
-          PACKAGE_NAME: ${{ matrix.pkg-name }}
-        run: |
-          python setup.py sdist
-          cp dist/* pypi/
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-
   install-pkg:
-    needs: build-pypi
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04, macOS-12, windows-2022]
-        pkg-name: ["app", "lite", "pytorch", ""]
+        pkg-name: ["app", "lite", "pytorch", "lightning"]
         python-version: ["3.7" , "3.10"]
     steps:
       - uses: actions/checkout@v3
+
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
@@ -86,27 +47,22 @@ jobs:
           python -m pytest setup_tools.py
 
       - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg-name }}' == 'pytorch' else 1))" >> $GITHUB_ENV
+
       - uses: ./.github/actions/pkg-check
         with:
           pkg-name: ${{ matrix.pkg-name }}
           nb-dirs: ${{ env.NB_DIRS }}
 
-      - uses: actions/download-artifact@v3
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-      - run: ls -l pypi/
       - uses: ./.github/actions/pkg-install
         with:
-          pip-flags: "--pre --find-links ../pypi/"
+          pkg-name: ${{ matrix.pkg-name }}
 
       - name: Run CLI
         # todo: add testing for `lightning_app`
-        if: ${{ matrix.pkg-name == '' }}
+        if: ${{ matrix.pkg-name == 'lightning' }}
         run: python -m lightning --version
 
       - name: DocTest package
         run: |
-          pip list
-          scope=$(python -c "lut = {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning'} ; print(lut.get('${{matrix.pkg-name}}', 'lightning'))")
-          python -m pytest src/${scope} --ignore-glob="**/cli/*-template/**"
+          PKG_NAME=$(python -c "print({'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])")
+          python -m pytest src/${PKG_NAME} --ignore-glob="**/cli/*-template/**"
diff --git a/.github/workflows/ci-pytorch-tests-slow.yml b/.github/workflows/ci-pytorch-tests-slow.yml
@@ -56,15 +56,6 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-py${{ matrix.python-version }}-
 
-    - name: Install lite for PL
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
-      env:
-        PACKAGE_NAME: "lite"
-      run: |
-        pip install -e . --upgrade --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
-        git checkout -- setup.py MANIFEST.in
-
     - name: Install package & dependencies
       env:
         PACKAGE_NAME: "pytorch"

diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml
@@ -124,16 +124,6 @@ jobs:
     - name: Switch PyTorch URL
       run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
 
-    - name: Install lite for PL
-      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
-      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
-      if: ${{ matrix.pkg-name != 'lightning' }}
-      env:
-        PACKAGE_NAME: "lite"
-      run: |
-        pip install -e . --upgrade --find-links ${TORCH_URL}
-        git checkout -- setup.py MANIFEST.in
-
     - name: Install package & dependencies
       env:
         PACKAGE_NAME: ${{ matrix.pkg-name }}