migrate Data subpackage (#19523)

Co-authored-by: Carlos Mocholí <[email protected]>
Co-authored-by: Jirka Borovec <[email protected]>
Co-authored-by: Jirka <[email protected]>

.actions/assistant.py

@@ -44,11 +44,7 @@
         "requirements/fabric/base.txt",
         "requirements/fabric/strategies.txt",
     ),
-    "data": (
-        "requirements/data/data.txt",
-        "requirements/data/cloud.txt",
-        "requirements/data/examples.txt",
-    ),
+    "data": ("requirements/data/data.txt",),
 }
 REQUIREMENT_FILES_ALL = list(chain(*REQUIREMENT_FILES.values()))
 

.github/CODEOWNERS

@@ -41,10 +41,8 @@
 /src/lightning/pytorch/core/hooks.py         @williamfalcon @tchaton @awaelchli @carmocca
 /src/lightning/pytorch/core/module.py        @williamfalcon @tchaton @awaelchli @carmocca
 
-# Data Utilities
-/examples/data/           @tchaton @nohalon @justusschock @lantiga
+# Lightning Data
 /src/lightning/data/      @tchaton
-/tests/tests_data         @tchaton
 
 # Lightning Fabric
 /src/lightning/fabric                       @awaelchli @carmocca @justusschock

.github/checkgroup.yml

@@ -150,29 +150,6 @@ subprojects:
       - "build-pl (3.10, 2.0, 11.8.0)"
       - "build-pl (3.10, 2.1, 12.1.0)"
 
-  # SECTIONS: lightning_data
-
-  - id: "lightning_data: CPU workflow"
-    paths:
-      - ".actions/*"
-      - "requirements/data/**"
-      - "src/lightning/data/**"
-      - "src/lightning_data/*"
-      - "src/lightning/__init__.py"
-      - "src/lightning/__setup__.py"
-      - "src/lightning/__version__.py"
-      - "tests/tests_data/**"
-      - "examples/data/**"
-      - "pyproject.toml" # includes pytest config
-      - ".github/workflows/ci-tests-data.yml"
-      - "!requirements/*/docs.txt"
-      - "!*.md"
-      - "!**/*.md"
-    checks:
-      - "data-cpu (macOS-11, lightning, 3.10, 2.1)"
-      - "data-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
-      - "data-cpu (windows-2022, lightning, 3.10, 2.1)"
-
   # SECTION: lightning_fabric
 
   - id: "lightning_fabric: CPU workflow"

.github/label-change.yml

@@ -14,7 +14,6 @@ data:
   - changed-files:
       - any-glob-to-any-file:
           - "src/lightning/data/**"
-          - "tests/tests_data/**"
           - "requirements/data/**"
 
 store:

.github/workflows/ci-pkg-install.yml

@@ -71,6 +71,7 @@ jobs:
           pkg-folder: dist/${{ env.PKG_DIR }}
           pkg-name: ${{ matrix.pkg-name }}
           pkg-extra: ${{ matrix.pkg-extra }}
+          pip-flags: "--pre" # todo: drop this when `lidata` is released (not as RC only)
 
       - name: Run CLI (via python)
         if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app' }}
@@ -100,13 +101,19 @@ jobs:
           rm -f ./*/__*.py
           rm -f ./**/__*.py
           mv lightning lit  # rename lightning folder to prevent accidental local imports
-      - name: drop App doctest
-        if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra != 'app' }}
+      - name: drop Secondary doctest
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
         working-directory: src/lit
         run: |
-          rm -rf app
-          rm -rf data
-          rm -rf store
+          items=("data" "store" "app")
+          for item in "${items[@]}"; do
+            if [[ "$item" == "${{ matrix.pkg-extra }}" ]]; then
+              echo "Skipping $item"
+              continue # Skip this iteration
+            fi
+            echo "Removing $item"
+            rm -rf $item
+          done
       - name: Install pytest doctest extension
         run: |
           pip install -q -r requirements/doctests.txt

.gitignore

@@ -195,7 +195,6 @@ our_model.tar
 test.png
 saved_models
 data/
-!src/lightning/data/
 !examples/data/
 !tests/tests_pytorch/utilities/data/
 !requirements/data/

.pre-commit-config.yaml

@@ -107,7 +107,6 @@ repos:
         exclude: |
           (?x)^(
               src/lightning/app/CHANGELOG.md|
-              src/lightning/data/CHANGELOG.md|
               src/lightning/fabric/CHANGELOG.md|
               src/lightning/pytorch/CHANGELOG.md|
               README.md

docs/source-pytorch/visualize/supported_exp_managers.rst

@@ -62,7 +62,7 @@ Here's the full documentation for the :class:`~lightning.pytorch.loggers.MLFlowL
 
 Neptune.ai
 ==========
-To use `Neptune.ai <https://neptune.ai/>`_ first install the neptune package:
+To use `Neptune.ai <https://www.neptune.ai/>`_ first install the neptune package:
 
 .. code-block:: bash
 

requirements/data/data.txt

@@ -1,8 +1,3 @@
-# NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
-#  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
+# NOTE: this is here only to expose `pip install lightning[data]`. we don't install or test it in this project's CI
 
-lightning-utilities >=0.8.0, <0.10.0
-# to be able to include also PL 2.0 and preserve `>` needed for CI min version bypass
-torch >0.14.0, <2.2.0
-lightning-cloud
-filelock
+litdata >= 0.2.0rc, <0.3.0

src/lightning/MANIFEST.in

@@ -3,4 +3,3 @@
 include src/lightning/version.info
 include src/lightning/py.typed
 include requirements/base.txt
-recursive-include requirements/data *.txt

src/lightning/__setup__.py

@@ -48,8 +48,6 @@ def _prepare_extras() -> Dict[str, Any]:
     extras["app-extra"] = extras["app-app"] + extras["app-cloud"] + extras["app-ui"] + extras["app-components"]
     extras["app-all"] = extras["app-extra"]
     extras["app-dev"] = extras["app-all"] + extras["app-test"]
-    extras["data-all"] = extras["data-data"] + extras["data-cloud"] + extras["data-examples"]
-    extras["data-dev"] = extras["data-all"] + extras["data-test"]
     extras["store-store"] = extras["app-app"]  # todo: consider cutting/leaning this dependency
 
     # merge per-project extras of the same category, e.g. `app-test` + `fabric-test`

src/lightning/data/__init__.py

@@ -1,15 +1,41 @@
+import sys
+
+from lightning_utilities.core.imports import RequirementCache
+
 from lightning.data.processing.functions import map, optimize, walk
 from lightning.data.streaming.combined import CombinedStreamingDataset
 from lightning.data.streaming.dataloader import StreamingDataLoader
 from lightning.data.streaming.dataset import StreamingDataset
 
-__all__ = [
-    "LightningDataset",
-    "StreamingDataset",
-    "CombinedStreamingDataset",
-    "StreamingDataLoader",
-    "LightningIterableDataset",
-    "map",
-    "optimize",
-    "walk",
-]
+__all__ = []
+
+if not RequirementCache("litdata"):
+    raise ModuleNotFoundError("Please, run `pip install litdata`")  # E111
+
+else:
+    import litdata
+
+    # Enable resolution at least for lower data namespace
+    sys.modules["lightning.data"] = litdata
+
+    from litdata.processing.functions import map, optimize, walk
+    from litdata.streaming.combined import CombinedStreamingDataset
+    from litdata.streaming.dataloader import StreamingDataLoader
+    from litdata.streaming.dataset import StreamingDataset
+
+    __all__ += [
+        "LightningDataset",
+        "StreamingDataset",
+        "CombinedStreamingDataset",
+        "StreamingDataLoader",
+        "LightningIterableDataset",
+        "map",
+        "optimize",
+        "walk",
+    ]
+
+# TODO: Move this to litdata
+if RequirementCache("lightning_sdk"):
+    from lightning_sdk import Machine
+
+    __all__ += ["Machine"]