Lightning-AI · justusschock · Dec 9, 2022 · Dec 7, 2022 · Dec 7, 2022 · Dec 7, 2022
diff --git a/.actions/setup_tools.py b/.actions/setup_tools.py
@@ -191,7 +191,7 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
         load_requirements(d, file_name="base.txt", unfreeze=not freeze_requirements)
         for d in glob.glob(os.path.join(req_dir, "*"))
         # skip empty folder as git artefacts, and resolving Will's special issue
-        if os.path.isdir(d) and len(glob.glob(os.path.join(d, "*"))) > 0
+        if os.path.isdir(d) and len(glob.glob(os.path.join(d, "*"))) > 0 and "__pycache__" not in d
     ]
     if not requires:
         return None

diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml
@@ -52,7 +52,7 @@ jobs:
   - job: App_cloud_e2e_testing
     pool: azure-cpus
     container:
-      image: mcr.microsoft.com/playwright/python:v1.27.1-focal
+      image: mcr.microsoft.com/playwright/python:v1.28.0-focal
       options: "--shm-size=4gb"
     strategy:
       matrix:

diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml
@@ -40,7 +40,7 @@ jobs:
     cancelTimeoutInMinutes: "2"
     pool: intel-hpus
     container:
-      image: "vault.habana.ai/gaudi-docker/1.7.0/ubuntu20.04/habanalabs/pytorch-installer-1.12.0:latest"
+      image: "vault.habana.ai/gaudi-docker/1.7.1/ubuntu20.04/habanalabs/pytorch-installer-1.13.0:latest"
       options: "--runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host --shm-size=4g -v /usr/bin/docker:/tmp/docker:ro"
     workspace:
       clean: all

diff --git a/.github/actions/pkg-publish/action.yml b/.github/actions/pkg-publish/action.yml
@@ -30,7 +30,7 @@ runs:
     if: inputs.pypi-test-token != ''
     with:
       user: __token__
-      password: ${{ secrets.test_pypi_token_lai }}
+      password: ${{ inputs.pypi-test-token }}
       repository_url: https://test.pypi.org/legacy/
       packages_dir: pypi/
       verbose: true

diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml
@@ -94,7 +94,7 @@ jobs:
 
       - name: Adjust tests
         if: ${{ matrix.pkg-name == 'lightning' }}
-        run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
+        run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app,lightning_lite,pytorch_lightning" --target_import="lightning.app,lightning.lite,lightning.pytorch"
 
       - name: Adjust examples
         if: ${{ matrix.pkg-name != 'lightning' }}

diff --git a/.github/workflows/legacy-checkpoints.yml b/.github/workflows/legacy-checkpoints.yml
@@ -72,6 +72,7 @@ jobs:
       working-directory: ./
       env:
         PACKAGE_NAME: pytorch
+        FREEZE_REQUIREMENTS: 1
       run: |
         pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html
         pip list

diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml
@@ -11,9 +11,6 @@ defaults:
   run:
     shell: bash
 
-env:
-  PUBLISH: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
-
 jobs:
   init:
     runs-on: ubuntu-20.04
@@ -184,7 +181,7 @@ jobs:
 
   publish-packages:
     runs-on: ubuntu-20.04
-    needs: waiting
+    needs: [build-packages, waiting]
     if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
     steps:
     - uses: actions/checkout@v3
@@ -215,8 +212,8 @@ jobs:
     needs: [build-packages]
     uses: ./.github/workflows/legacy-checkpoints.yml
     with:
-      push_to_s3: ${{ env.PUBLISH }}
-      create_pr: ${{ env.PUBLISH }}
+      push_to_s3: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
+      create_pr: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_KEY_ID: ${{ secrets.AWS_SECRET_KEY_ID }}
diff --git a/dockers/ci-runner-hpu/Dockerfile b/dockers/ci-runner-hpu/Dockerfile
@@ -16,8 +16,8 @@
 #        gaudi-docker-agent:latest
 
 ARG DIST="latest"
-ARG GAUDI_VERSION="1.7.0"
-ARG PYTORCH_INSTALLER_VERSION="1.12.0"
+ARG GAUDI_VERSION="1.7.1"
+ARG PYTORCH_INSTALLER_VERSION="1.13.0"
 FROM vault.habana.ai/gaudi-docker/${GAUDI_VERSION}/ubuntu20.04/habanalabs/pytorch-installer-${PYTORCH_INSTALLER_VERSION}:${DIST}
 
 LABEL maintainer="https://vault.habana.ai/"

diff --git a/docs/source-app/api_reference/runners.rst b/docs/source-app/api_reference/runners.rst
@@ -18,5 +18,4 @@ ______________
     :template: classtemplate.rst
 
     ~cloud.CloudRuntime
-    ~singleprocess.SingleProcessRuntime
     ~multiprocess.MultiProcessRuntime
diff --git a/docs/source-app/api_reference/storage.rst b/docs/source-app/api_reference/storage.rst
@@ -20,6 +20,7 @@ ______________
     ~path.Path
     ~drive.Drive
     ~payload.Payload
+    ~mount.Mount
 
 ----
 
@@ -56,6 +57,14 @@ Learn more about Storage
    :height: 180
    :tag: Intermediate
 
+.. displayitem::
+   :header: The Mount Object.
+   :description: Mount an AWS S3 Bucket When Running on the Cloud.
+   :col_css: col-md-4
+   :button_link:  ../workflows/mount_aws_s3_bucket.html
+   :height: 180
+   :tag: Intermediate
+
 .. raw:: html
 
         </div>

diff --git a/docs/source-app/api_references.rst b/docs/source-app/api_references.rst
@@ -32,11 +32,20 @@ ___________________
     :nosignatures:
     :template: classtemplate_no_index.rst
 
+    ~database.client.DatabaseClient
+    ~database.server.Database
     ~python.popen.PopenPythonScript
     ~python.tracer.TracerPythonScript
     ~training.LightningTrainerScript
     ~serve.gradio.ServeGradio
     ~serve.serve.ModelInferenceAPI
+    ~serve.python_server.PythonServer
+    ~serve.streamlit.ServeStreamlit
+    ~multi_node.base.MultiNode
+    ~multi_node.lite.LiteMultiNode
+    ~multi_node.pytorch_spawn.PyTorchSpawnMultiNode
+    ~multi_node.trainer.LightningTrainerMultiNode
+    ~auto_scaler.AutoScaler
 
 ----
 
@@ -71,6 +80,7 @@ _______
     ~path.Path
     ~drive.Drive
     ~payload.Payload
+    ~mount.Mount
 
 Learn more about :ref:`Storage <storage>`.
 
@@ -87,5 +97,19 @@ _______
     :template: classtemplate_no_index.rst
 
     ~cloud.CloudRuntime
-    ~singleprocess.SingleProcessRuntime
     ~multiprocess.MultiProcessRuntime
+
+----
+
+lightning_app.utilities.packaging
+_________________________________
+
+.. currentmodule:: lightning_app.utilities.packaging
+
+.. autosummary::
+    :toctree: generated/
+    :nosignatures:
+    :template: classtemplate_no_index.rst
+
+    ~cloud_compute.CloudCompute
+    ~build_config.BuildConfig
diff --git a/docs/source-app/glossary/index.rst b/docs/source-app/glossary/index.rst
@@ -112,6 +112,13 @@ Glossary
    :button_link: ../core_api/lightning_app/index.html
    :height: 100
 
+.. displayitem::
+   :header: Mounts
+   :description: Mount Cloud Data
+   :col_css: col-md-6
+   :button_link: mount.html
+   :height: 180
+
 .. displayitem::
    :header: Sharing Components
    :description: Let's create an ecosystem altogether

diff --git a/docs/source-app/glossary/mount.rst b/docs/source-app/glossary/mount.rst
@@ -0,0 +1 @@
+.. include:: ../workflows/mount_cloud_object_store.rst
diff --git a/docs/source-app/testing.rst b/docs/source-app/testing.rst
@@ -120,7 +120,6 @@ We provide ``application_testing`` as a helper funtion to get your application u
        os.path.join(_PROJECT_ROOT, "examples/app_v0/app.py"),
        "--blocking",
        "False",
-       "--multiprocess",
        "--open-ui",
        "False",
    ]
@@ -129,9 +128,7 @@ First in the list for ``command_line`` is the location of your script. It is an
 
 Next there are a couple of options you can leverage:
 
-
 * ``blocking`` - Blocking is an app status that says "Do not run until I click run in the UI". For our integration test, since we are not using the UI, we are setting this to "False".
-* ``multiprocess/singleprocess`` - This is the runtime your app is expected to run under.
 * ``open-ui`` - We set this to false since this is the routine that opens a browser for your local execution.
 
 Once you have your commandline ready, you will then be able to kick off the test and gather results:

diff --git a/docs/source-app/workflows/index.rst b/docs/source-app/workflows/index.rst
@@ -187,6 +187,14 @@ How to:
    :button_link: ssh/index.html
    :height: 180
 
+.. displayitem::
+   :header: Mount Cloud Data
+   :description: Learn how Lightning Mounts are used to make the contents of an cloud object store bucket available on disk when running in the cloud.
+   :col_css: col-md-4
+   :button_link: mount_cloud_object_store.html
+   :height: 180
+
+
 
 .. raw:: html
 

diff --git a/docs/source-app/workflows/mount_cloud_object_store.rst b/docs/source-app/workflows/mount_cloud_object_store.rst
@@ -0,0 +1,141 @@
+:orphan:
+
+##############
+Add Cloud Data
+##############
+
+**Audience:** Users who want to read files stored in a Cloud Object Bucket in an app.
+
+******************************
+Mounting Public AWS S3 Buckets
+******************************
+
+===================
+Add Mount to a Work
+===================
+
+To mount data from a cloud bucket to your app compute, initialize a :class:`~lightning_app.storage.mount.Mount`
+object with the source path of the s3 bucket and the absolute directory path where it should be mounted and
+pass the :class:`~lightning_app.storage.mount.Mount` to the :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute`
+of the :class:`~lightning_app.core.work.LightningWork` it should be mounted on.
+
+In this example, we will mount an S3 bucket: ``s3://ryft-public-sample-data/esRedditJson/`` to ``/content/esRedditJson/``.
+
+.. code-block:: python
+
+    from lightning_app import CloudCompute
+    from lightning_app.storage import Mount
+
+    self.my_work = MyWorkClass(
+        cloud_compute=CloudCompute(
+            mounts=Mount(
+                source="s3://ryft-public-sample-data/esRedditJson/",
+                mount_path="/content/esRedditJson/",
+            ),
+        )
+    )
+
+You can also pass multiple mounts to a single work by passing a ``List[Mount(...), ...]`` to the
+``CloudCompute(mounts=...)`` argument.
+
+.. note::
+
+    * Mounts supported up to 1 Million files, 5GB per file. Need larger mounts? Contact [email protected]
+    * When adding multiple mounts, each one should have a unique ``mount_path``.
+    * A maximum of 10 :class:`~lightning_app.storage.mount.Mount`\s can be added to a :class:`~lightning_app.core.work.LightningWork`.
+
+=======================
+Read Files From a Mount
+=======================
+
+Once a :class:`~lightning_app.storage.mount.Mount` object is passed to :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute`,
+you can access, list, or read any file from the mount under the specified ``mount_path``, just like you would if it
+was on your local machine.
+
+Assuming your ``mount_path`` is ``"/content/esRedditJson/"`` you can do the following:
+
+----------
+Read Files
+----------
+
+.. code-block:: python
+
+    with open("/content/esRedditJson/esRedditJson1", "r") as f:
+        some_data = f.read()
+
+    # do something with "some_data"...
+
+----------
+List Files
+----------
+
+.. code-block:: python
+
+    files = os.listdir("/content/esRedditJson/")
+
+--------------------
+See the Full Example
+--------------------
+
+.. code-block:: python
+    :emphasize-lines: 10,15
+
+    import os
+
+    import lightning as L
+    from lightning_app import CloudCompute
+    from lightning_app.storage import Mount
+
+    class ReadMount(L.LightningWork):
+       def run(self):
+           # Print a list of files stored in the mounted S3 Bucket.
+           files = os.listdir("/content/esRedditJson/")
+           for file in files:
+               print(file)
+
+           # Read the contents of a particular file in the bucket "esRedditJson1"
+           with open("/content/esRedditJson/esRedditJson1", "r") as f:
+               some_data = f.read()
+               # do something with "some_data"...
+
+    class Flow(L.LightningFlow):
+       def __init__(self):
+           super().__init__()
+           self.my_work = ReadMount(
+               cloud_compute=CloudCompute(
+                   mounts=Mount(
+                       source="s3://ryft-public-sample-data/esRedditJson/",
+                       mount_path="/content/esRedditJson/",
+                   ),
+               )
+           )
+
+       def run(self):
+           self.my_work.run()
+
+.. note::
+
+    When running a Lighting App on your local machine, any :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute`
+    configuration (including a :class:`~lightning_app.storage.mount.Mount`) is ignored at runtime. If you need access to
+    these files on your local disk, you should download a copy of them to your machine.
+
+.. note::
+
+    Mounted files from an S3 bucket are ``read-only``. Any modifications, additions, or deletions
+    to files in the mounted directory will not be reflected in the cloud object store.
+
+----
+
+**********************************************
+Mounting Private AWS S3 Buckets - Coming Soon!
+**********************************************
+
+We'll Let you know when this feature is ready!
+
+----
+
+************************************************
+Mounting Google Cloud GCS Buckets - Coming Soon!
+************************************************
+
+We'll Let you know when this feature is ready!
diff --git a/docs/source-pytorch/accelerators/hpu_basic.rst b/docs/source-pytorch/accelerators/hpu_basic.rst
@@ -113,4 +113,3 @@ Known limitations
 -----------------
 
 * `Habana dataloader <https://docs.habana.ai/en/latest/PyTorch_User_Guide/PyTorch_User_Guide.html#habana-data-loader>`__ is not supported.
-* :func:`torch.inference_mode` is not supported
diff --git a/docs/source-pytorch/accelerators/hpu_intermediate.rst b/docs/source-pytorch/accelerators/hpu_intermediate.rst
@@ -96,4 +96,4 @@ The below snippet shows how DeviceStatsMonitor can be enabled.
     device_stats = DeviceStatsMonitor()
     trainer = Trainer(accelerator="hpu", callbacks=[device_stats])
 
-For more details, please refer to `Memory Stats APIs <https://docs.habana.ai/en/v1.5.0/PyTorch/PyTorch_User_Guide/Python_Packages.html#memory-stats-apis>`__.
+For more details, please refer to `Memory Stats APIs <https://docs.habana.ai/en/latest/PyTorch/PyTorch_User_Guide/Python_Packages.html#memory-stats-apis>`__.
diff --git a/docs/source-pytorch/advanced/model_parallel.rst b/docs/source-pytorch/advanced/model_parallel.rst
@@ -424,10 +424,9 @@ You can customize the strategy configuration by adjusting the arguments of :clas
 
     from pytorch_lightning import Trainer
     from pytorch_lightning.strategies import DDPFullyShardedNativeStrategy
-    from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload
 
 
-    native_fsdp = DDPFullyShardedNativeStrategy(cpu_offload=CPUOffload(offload_params=True))
+    native_fsdp = DDPFullyShardedNativeStrategy(cpu_offload=True)
     trainer = pl.Trainer(strategy=native_fsdp, accelerator="gpu", devices=4)
 
 

diff --git a/docs/source-pytorch/clouds/cluster_intermediate_1.rst b/docs/source-pytorch/clouds/cluster_intermediate_1.rst
@@ -24,7 +24,7 @@ PyTorch Lightning follows the design of `PyTorch distributed communication packa
 
 - *MASTER_PORT* - required; has to be a free port on machine with NODE_RANK 0
 - *MASTER_ADDR* - required (except for NODE_RANK 0); address of NODE_RANK 0 node
-- *WORLD_SIZE* - required; how many nodes are in the cluster
+- *WORLD_SIZE* - required; the total number of GPUs/processes that you will use
 - *NODE_RANK* - required; id of the node in the cluster
 
 .. _training_script_setup:

diff --git a/examples/app_dag/requirements.txt b/examples/app_dag/requirements.txt
@@ -1,2 +1,2 @@
-sklearn
+scikit-learn
 pandas
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		.. include:: ../workflows/mount_cloud_object_store.rst
Original file line number	Diff line number	Diff line change
Expand Up		@@ -113,4 +113,3 @@ Known limitations
		-----------------

		* `Habana dataloader <https://docs.habana.ai/en/latest/PyTorch_User_Guide/PyTorch_User_Guide.html#habana-data-loader>`__ is not supported.
		* :func:`torch.inference_mode` is not supported