From 2e36719d73060ce97de9503564d9d7af2bca5e7e Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 8 Feb 2024 08:02:08 +0000 Subject: [PATCH 1/8] Lightning Data: Refactor files (#19424) (cherry picked from commit ac9d63f4eb3c6055c7fd67d2430e6d171e9a59aa) --- src/lightning/data/__init__.py | 2 +- .../data_processor.py | 0 .../{streaming => processing}/functions.py | 2 +- src/lightning/data/processing/readers.py | 2 +- src/lightning/data/streaming/__init__.py | 4 - src/lightning/data/streaming/shuffle.py | 74 +----------------- src/lightning/data/utilities/shuffle.py | 78 +++++++++++++++++++ .../test_data_processor.py | 15 ++-- .../test_functions.py | 2 +- tests/tests_data/streaming/test_dataset.py | 3 +- tests/tests_data/streaming/test_shuffle.py | 2 +- 11 files changed, 94 insertions(+), 90 deletions(-) rename src/lightning/data/{streaming => processing}/data_processor.py (100%) rename src/lightning/data/{streaming => processing}/functions.py (99%) create mode 100644 src/lightning/data/utilities/shuffle.py rename tests/tests_data/{streaming => processing}/test_data_processor.py (98%) rename tests/tests_data/{streaming => processing}/test_functions.py (94%) diff --git a/src/lightning/data/__init__.py b/src/lightning/data/__init__.py index 88b384e8a227b..21cd09bbe8d98 100644 --- a/src/lightning/data/__init__.py +++ b/src/lightning/data/__init__.py @@ -1,7 +1,7 @@ +from lightning.data.processing.functions import map, optimize, walk from lightning.data.streaming.combined import CombinedStreamingDataset from lightning.data.streaming.dataloader import StreamingDataLoader from lightning.data.streaming.dataset import StreamingDataset -from lightning.data.streaming.functions import map, optimize, walk __all__ = [ "LightningDataset", diff --git a/src/lightning/data/streaming/data_processor.py b/src/lightning/data/processing/data_processor.py similarity index 100% rename from src/lightning/data/streaming/data_processor.py rename to src/lightning/data/processing/data_processor.py diff --git a/src/lightning/data/streaming/functions.py b/src/lightning/data/processing/functions.py similarity index 99% rename from src/lightning/data/streaming/functions.py rename to src/lightning/data/processing/functions.py index 7bc865ccb8d85..6939418ad617c 100644 --- a/src/lightning/data/streaming/functions.py +++ b/src/lightning/data/processing/functions.py @@ -22,9 +22,9 @@ import torch +from lightning.data.processing.data_processor import DataChunkRecipe, DataProcessor, DataTransformRecipe from lightning.data.processing.readers import BaseReader from lightning.data.streaming.constants import _IS_IN_STUDIO, _TORCH_GREATER_EQUAL_2_1_0 -from lightning.data.streaming.data_processor import DataChunkRecipe, DataProcessor, DataTransformRecipe from lightning.data.streaming.resolver import ( Dir, _assert_dir_has_index_file, diff --git a/src/lightning/data/processing/readers.py b/src/lightning/data/processing/readers.py index d9a5b293bdba0..8519795b6207c 100644 --- a/src/lightning/data/processing/readers.py +++ b/src/lightning/data/processing/readers.py @@ -5,8 +5,8 @@ from lightning_utilities.core.imports import RequirementCache -from lightning.data.streaming.shuffle import _associate_chunks_and_internals_to_ranks from lightning.data.utilities.env import _DistributedEnv +from lightning.data.utilities.shuffle import _associate_chunks_and_internals_to_ranks _POLARS_AVAILABLE = RequirementCache("polars") _PYARROW_AVAILABLE = RequirementCache("pyarrow") diff --git a/src/lightning/data/streaming/__init__.py b/src/lightning/data/streaming/__init__.py index 55b88dc2758c1..2e6c49cfe6e59 100644 --- a/src/lightning/data/streaming/__init__.py +++ b/src/lightning/data/streaming/__init__.py @@ -13,18 +13,14 @@ from lightning.data.streaming.cache import Cache from lightning.data.streaming.combined import CombinedStreamingDataset -from lightning.data.streaming.data_processor import DataChunkRecipe, DataProcessor, DataTransformRecipe from lightning.data.streaming.dataloader import StreamingDataLoader from lightning.data.streaming.dataset import StreamingDataset from lightning.data.streaming.item_loader import TokensLoader __all__ = [ "Cache", - "DataProcessor", "StreamingDataset", "CombinedStreamingDataset", "StreamingDataLoader", - "DataTransformRecipe", - "DataChunkRecipe", "TokensLoader", ] diff --git a/src/lightning/data/streaming/shuffle.py b/src/lightning/data/streaming/shuffle.py index 3c6f91a97e3db..b0a48bd728eb4 100644 --- a/src/lightning/data/streaming/shuffle.py +++ b/src/lightning/data/streaming/shuffle.py @@ -19,6 +19,7 @@ from lightning.data.streaming import Cache from lightning.data.utilities.env import _DistributedEnv +from lightning.data.utilities.shuffle import _associate_chunks_and_internals_to_ranks, _intra_node_chunk_shuffle class Shuffle(ABC): @@ -129,76 +130,3 @@ def get_chunks_and_intervals_per_ranks(self, distributed_env: _DistributedEnv, c def __call__(self, array: np.ndarray, num_chunks: int, current_epoch: int, chunk_index: int) -> List[int]: return np.random.RandomState([self.seed, num_chunks * current_epoch, chunk_index]).permutation(array).tolist() - - -def _intra_node_chunk_shuffle( - distributed_env: _DistributedEnv, - chunks_per_ranks: List[List[int]], - seed: int, - current_epoch: int, -) -> List[int]: - chunk_indexes_per_nodes: Any = [[] for _ in range(distributed_env.num_nodes)] - for rank, chunks_per_rank in enumerate(chunks_per_ranks): - chunk_indexes_per_nodes[0 if distributed_env.num_nodes == 1 else rank // distributed_env.num_nodes].extend( - chunks_per_rank - ) - - # shuffle the chunks associated to the node - for i in range(len(chunk_indexes_per_nodes)): - # permute the indexes within the node - chunk_indexes_per_nodes[i] = np.random.RandomState(seed=seed + current_epoch).permutation( - chunk_indexes_per_nodes[i] - ) - - return [index for chunks in chunk_indexes_per_nodes for index in chunks] - - -def _associate_chunks_and_internals_to_ranks( - distributed_env: _DistributedEnv, - indexes: Any, - chunk_intervals: Any, - drop_last: bool, -) -> Tuple[List[List[int]], List[Any]]: - num_items = sum([(interval[-1] - interval[0]) for interval in chunk_intervals]) - num_items_per_ranks: List[int] = [ - num_items // distributed_env.world_size + num_items % distributed_env.world_size - if rank == distributed_env.world_size - 1 and not drop_last - else num_items // distributed_env.world_size - for rank in range(distributed_env.world_size) - ] - chunks_per_ranks: List[List[int]] = [[] for _ in range(distributed_env.world_size)] - intervals_per_ranks: List[List[List[int]]] = [[] for _ in range(distributed_env.world_size)] - - # 4. Assign the chunk & intervals to each rank - for chunk_index, chunk_interval in zip(indexes, chunk_intervals): - rank = 0 - - while True: - if rank == len(num_items_per_ranks): - break - - items_left_to_assign = num_items_per_ranks[rank] - - if items_left_to_assign == 0: - rank += 1 - continue - - items_in_chunk = chunk_interval[-1] - chunk_interval[0] - - if items_in_chunk == 0: - break - - if items_in_chunk > items_left_to_assign: - chunks_per_ranks[rank].append(chunk_index) - begin, end = chunk_interval - intervals_per_ranks[rank].append([begin, begin + items_left_to_assign]) - chunk_interval = (begin + items_left_to_assign, end) - num_items_per_ranks[rank] = 0 - rank += 1 - else: - chunks_per_ranks[rank].append(chunk_index) - intervals_per_ranks[rank].append(chunk_interval) - num_items_per_ranks[rank] -= items_in_chunk - break - - return chunks_per_ranks, intervals_per_ranks diff --git a/src/lightning/data/utilities/shuffle.py b/src/lightning/data/utilities/shuffle.py new file mode 100644 index 0000000000000..8be6563096ed6 --- /dev/null +++ b/src/lightning/data/utilities/shuffle.py @@ -0,0 +1,78 @@ +from typing import Any, List, Tuple + +import numpy as np + +from lightning.data.utilities.env import _DistributedEnv + + +def _intra_node_chunk_shuffle( + distributed_env: _DistributedEnv, + chunks_per_ranks: List[List[int]], + seed: int, + current_epoch: int, +) -> List[int]: + chunk_indexes_per_nodes: Any = [[] for _ in range(distributed_env.num_nodes)] + for rank, chunks_per_rank in enumerate(chunks_per_ranks): + chunk_indexes_per_nodes[0 if distributed_env.num_nodes == 1 else rank // distributed_env.num_nodes].extend( + chunks_per_rank + ) + + # shuffle the chunks associated to the node + for i in range(len(chunk_indexes_per_nodes)): + # permute the indexes within the node + chunk_indexes_per_nodes[i] = np.random.RandomState(seed=seed + current_epoch).permutation( + chunk_indexes_per_nodes[i] + ) + + return [index for chunks in chunk_indexes_per_nodes for index in chunks] + + +def _associate_chunks_and_internals_to_ranks( + distributed_env: _DistributedEnv, + indexes: Any, + chunk_intervals: Any, + drop_last: bool, +) -> Tuple[List[List[int]], List[Any]]: + num_items = sum([(interval[-1] - interval[0]) for interval in chunk_intervals]) + num_items_per_ranks: List[int] = [ + num_items // distributed_env.world_size + num_items % distributed_env.world_size + if rank == distributed_env.world_size - 1 and not drop_last + else num_items // distributed_env.world_size + for rank in range(distributed_env.world_size) + ] + chunks_per_ranks: List[List[int]] = [[] for _ in range(distributed_env.world_size)] + intervals_per_ranks: List[List[List[int]]] = [[] for _ in range(distributed_env.world_size)] + + # 4. Assign the chunk & intervals to each rank + for chunk_index, chunk_interval in zip(indexes, chunk_intervals): + rank = 0 + + while True: + if rank == len(num_items_per_ranks): + break + + items_left_to_assign = num_items_per_ranks[rank] + + if items_left_to_assign == 0: + rank += 1 + continue + + items_in_chunk = chunk_interval[-1] - chunk_interval[0] + + if items_in_chunk == 0: + break + + if items_in_chunk > items_left_to_assign: + chunks_per_ranks[rank].append(chunk_index) + begin, end = chunk_interval + intervals_per_ranks[rank].append([begin, begin + items_left_to_assign]) + chunk_interval = (begin + items_left_to_assign, end) + num_items_per_ranks[rank] = 0 + rank += 1 + else: + chunks_per_ranks[rank].append(chunk_index) + intervals_per_ranks[rank].append(chunk_interval) + num_items_per_ranks[rank] -= items_in_chunk + break + + return chunks_per_ranks, intervals_per_ranks diff --git a/tests/tests_data/streaming/test_data_processor.py b/tests/tests_data/processing/test_data_processor.py similarity index 98% rename from tests/tests_data/streaming/test_data_processor.py rename to tests/tests_data/processing/test_data_processor.py index 671917abe18b1..06420130321a9 100644 --- a/tests/tests_data/streaming/test_data_processor.py +++ b/tests/tests_data/processing/test_data_processor.py @@ -10,10 +10,9 @@ import pytest import torch from lightning import seed_everything -from lightning.data.streaming import data_processor as data_processor_module -from lightning.data.streaming import functions, resolver -from lightning.data.streaming.cache import Cache, Dir -from lightning.data.streaming.data_processor import ( +from lightning.data.processing import data_processor as data_processor_module +from lightning.data.processing import functions +from lightning.data.processing.data_processor import ( DataChunkRecipe, DataProcessor, DataTransformRecipe, @@ -26,7 +25,9 @@ _wait_for_disk_usage_higher_than_threshold, _wait_for_file_to_exist, ) -from lightning.data.streaming.functions import LambdaDataTransformRecipe, map, optimize +from lightning.data.processing.functions import LambdaDataTransformRecipe, map, optimize +from lightning.data.streaming import resolver +from lightning.data.streaming.cache import Cache, Dir from lightning_utilities.core.imports import RequirementCache _PIL_AVAILABLE = RequirementCache("PIL") @@ -162,7 +163,7 @@ def fn(*_, **__): @pytest.mark.skipif(condition=sys.platform == "win32", reason="Not supported on windows") -@mock.patch("lightning.data.streaming.data_processor._wait_for_disk_usage_higher_than_threshold") +@mock.patch("lightning.data.processing.data_processor._wait_for_disk_usage_higher_than_threshold") def test_download_data_target(wait_for_disk_usage_higher_than_threshold_mock, tmpdir): input_dir = os.path.join(tmpdir, "input_dir") os.makedirs(input_dir, exist_ok=True) @@ -201,7 +202,7 @@ def fn(*_, **__): def test_wait_for_disk_usage_higher_than_threshold(): disk_usage_mock = mock.Mock(side_effect=[mock.Mock(free=10e9), mock.Mock(free=10e9), mock.Mock(free=10e11)]) - with mock.patch("lightning.data.streaming.data_processor.shutil.disk_usage", disk_usage_mock): + with mock.patch("lightning.data.processing.data_processor.shutil.disk_usage", disk_usage_mock): _wait_for_disk_usage_higher_than_threshold("/", 10, sleep_time=0) assert disk_usage_mock.call_count == 3 diff --git a/tests/tests_data/streaming/test_functions.py b/tests/tests_data/processing/test_functions.py similarity index 94% rename from tests/tests_data/streaming/test_functions.py rename to tests/tests_data/processing/test_functions.py index 10bf40caf7c2f..d0b581130d928 100644 --- a/tests/tests_data/streaming/test_functions.py +++ b/tests/tests_data/processing/test_functions.py @@ -4,7 +4,7 @@ import pytest from lightning.data import walk -from lightning.data.streaming.functions import _get_input_dir +from lightning.data.processing.functions import _get_input_dir @pytest.mark.skipif(sys.platform == "win32", reason="currently not supported for windows.") diff --git a/tests/tests_data/streaming/test_dataset.py b/tests/tests_data/streaming/test_dataset.py index db294f6ea564f..e48db3fab9f30 100644 --- a/tests/tests_data/streaming/test_dataset.py +++ b/tests/tests_data/streaming/test_dataset.py @@ -20,7 +20,8 @@ import pytest import torch from lightning import seed_everything -from lightning.data.streaming import Cache, functions +from lightning.data.processing import functions +from lightning.data.streaming import Cache from lightning.data.streaming.dataloader import StreamingDataLoader from lightning.data.streaming.dataset import ( _INDEX_FILENAME, diff --git a/tests/tests_data/streaming/test_shuffle.py b/tests/tests_data/streaming/test_shuffle.py index 519c61f8239a0..cb451ce73a1ec 100644 --- a/tests/tests_data/streaming/test_shuffle.py +++ b/tests/tests_data/streaming/test_shuffle.py @@ -1,5 +1,5 @@ -from lightning.data.streaming.shuffle import _associate_chunks_and_internals_to_ranks, _intra_node_chunk_shuffle from lightning.data.utilities.env import _DistributedEnv +from lightning.data.utilities.shuffle import _associate_chunks_and_internals_to_ranks, _intra_node_chunk_shuffle def test_intra_node_chunk_shuffle(): From e30d787f8adccb221fa89010e77827c083d06b98 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 8 Feb 2024 11:14:57 +0000 Subject: [PATCH 2/8] Add DNS optimize support (#19429) * update * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update * update * update * update * update * update --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: thomas (cherry picked from commit 4c2fc3b0cb293a776a39f69e59e6bdc03adc2ef2) --- .../data/{streaming => }/constants.py | 0 .../data/processing/data_processor.py | 10 ++-- src/lightning/data/processing/dns.py | 47 +++++++++++++++++++ src/lightning/data/processing/functions.py | 25 ++++++---- src/lightning/data/streaming/cache.py | 2 +- src/lightning/data/streaming/client.py | 2 +- src/lightning/data/streaming/config.py | 2 +- src/lightning/data/streaming/dataloader.py | 2 +- src/lightning/data/streaming/dataset.py | 4 +- src/lightning/data/streaming/downloader.py | 2 +- src/lightning/data/streaming/item_loader.py | 2 +- src/lightning/data/streaming/reader.py | 2 +- src/lightning/data/streaming/serializers.py | 2 +- src/lightning/data/streaming/writer.py | 2 +- tests/tests_data/processing/test_dns.py | 33 +++++++++++++ 15 files changed, 111 insertions(+), 26 deletions(-) rename src/lightning/data/{streaming => }/constants.py (100%) create mode 100644 src/lightning/data/processing/dns.py create mode 100644 tests/tests_data/processing/test_dns.py diff --git a/src/lightning/data/streaming/constants.py b/src/lightning/data/constants.py similarity index 100% rename from src/lightning/data/streaming/constants.py rename to src/lightning/data/constants.py diff --git a/src/lightning/data/processing/data_processor.py b/src/lightning/data/processing/data_processor.py index 0449a1218623a..3a757857b08f8 100644 --- a/src/lightning/data/processing/data_processor.py +++ b/src/lightning/data/processing/data_processor.py @@ -20,11 +20,7 @@ from tqdm.auto import tqdm as _tqdm from lightning import seed_everything -from lightning.data.processing.readers import BaseReader -from lightning.data.streaming import Cache -from lightning.data.streaming.cache import Dir -from lightning.data.streaming.client import S3Client -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _BOTO3_AVAILABLE, _DEFAULT_FAST_DEV_RUN_ITEMS, _INDEX_FILENAME, @@ -32,6 +28,10 @@ _LIGHTNING_CLOUD_LATEST, _TORCH_GREATER_EQUAL_2_1_0, ) +from lightning.data.processing.readers import BaseReader +from lightning.data.streaming import Cache +from lightning.data.streaming.cache import Dir +from lightning.data.streaming.client import S3Client from lightning.data.streaming.resolver import _resolve_dir from lightning.data.utilities.broadcast import broadcast_object from lightning.data.utilities.packing import _pack_greedily diff --git a/src/lightning/data/processing/dns.py b/src/lightning/data/processing/dns.py new file mode 100644 index 0000000000000..f1ca83dbc0e2e --- /dev/null +++ b/src/lightning/data/processing/dns.py @@ -0,0 +1,47 @@ +from contextlib import contextmanager +from subprocess import Popen +from typing import Any + +from lightning.data.constants import _IS_IN_STUDIO + + +@contextmanager +def optimize_dns_context(enable: bool) -> Any: + optimize_dns(enable) + try: + yield + optimize_dns(False) # always disable the optimize DNS + except Exception as e: + optimize_dns(False) # always disable the optimize DNS + raise e + +def optimize_dns(enable: bool) -> None: + if not _IS_IN_STUDIO: + return + + with open("/etc/resolv.conf") as f: + lines = f.readlines() + + if ( + (enable and any("127.0.0.53" in line for line in lines)) + or (not enable and any("127.0.0.1" in line for line in lines)) + ): # noqa E501 + Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing.dns import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() # noqa E501 + +def _optimize_dns(enable: bool) -> None: + with open("/etc/resolv.conf") as f: + lines = f.readlines() + + write_lines = [] + for line in lines: + if "nameserver 127" in line: + if enable: + write_lines.append('nameserver 127.0.0.1\n') + else: + write_lines.append('nameserver 127.0.0.53\n') + else: + write_lines.append(line) + + with open("/etc/resolv.conf", "w") as f: + for line in write_lines: + f.write(line) diff --git a/src/lightning/data/processing/functions.py b/src/lightning/data/processing/functions.py index 6939418ad617c..00905aa40dcd4 100644 --- a/src/lightning/data/processing/functions.py +++ b/src/lightning/data/processing/functions.py @@ -22,9 +22,10 @@ import torch +from lightning.data.constants import _IS_IN_STUDIO, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.processing.data_processor import DataChunkRecipe, DataProcessor, DataTransformRecipe +from lightning.data.processing.dns import optimize_dns_context from lightning.data.processing.readers import BaseReader -from lightning.data.streaming.constants import _IS_IN_STUDIO, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.resolver import ( Dir, _assert_dir_has_index_file, @@ -218,7 +219,8 @@ def map( weights=weights, reader=reader, ) - return data_processor.run(LambdaDataTransformRecipe(fn, inputs)) + with optimize_dns_context(True): + return data_processor.run(LambdaDataTransformRecipe(fn, inputs)) return _execute( f"data-prep-map-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}", num_nodes, @@ -303,15 +305,18 @@ def optimize( reorder_files=reorder_files, reader=reader, ) - return data_processor.run( - LambdaDataChunkRecipe( - fn, - inputs, - chunk_size=chunk_size, - chunk_bytes=chunk_bytes, - compression=compression, + + with optimize_dns_context(True): + data_processor.run( + LambdaDataChunkRecipe( + fn, + inputs, + chunk_size=chunk_size, + chunk_bytes=chunk_bytes, + compression=compression, + ) ) - ) + return None return _execute( f"data-prep-optimize-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}", num_nodes, diff --git a/src/lightning/data/streaming/cache.py b/src/lightning/data/streaming/cache.py index 667409c81b8d8..305f393d276ac 100644 --- a/src/lightning/data/streaming/cache.py +++ b/src/lightning/data/streaming/cache.py @@ -15,7 +15,7 @@ import os from typing import Any, Dict, List, Optional, Tuple, Union -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _INDEX_FILENAME, _LIGHTNING_CLOUD_LATEST, _TORCH_GREATER_EQUAL_2_1_0, diff --git a/src/lightning/data/streaming/client.py b/src/lightning/data/streaming/client.py index c42d3af9f6ed1..4a572548854de 100644 --- a/src/lightning/data/streaming/client.py +++ b/src/lightning/data/streaming/client.py @@ -2,7 +2,7 @@ from time import time from typing import Any, Optional -from lightning.data.streaming.constants import _BOTO3_AVAILABLE +from lightning.data.constants import _BOTO3_AVAILABLE if _BOTO3_AVAILABLE: import boto3 diff --git a/src/lightning/data/streaming/config.py b/src/lightning/data/streaming/config.py index 386ea475e7681..4a5a4ba8c55b2 100644 --- a/src/lightning/data/streaming/config.py +++ b/src/lightning/data/streaming/config.py @@ -15,7 +15,7 @@ import os from typing import Any, Dict, List, Optional, Tuple -from lightning.data.streaming.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 +from lightning.data.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.downloader import get_downloader_cls from lightning.data.streaming.item_loader import BaseItemLoader, PyTreeLoader, TokensLoader from lightning.data.streaming.sampler import ChunkedIndex diff --git a/src/lightning/data/streaming/dataloader.py b/src/lightning/data/streaming/dataloader.py index 942d2f98a3090..04793296ca30f 100644 --- a/src/lightning/data/streaming/dataloader.py +++ b/src/lightning/data/streaming/dataloader.py @@ -33,13 +33,13 @@ ) from torch.utils.data.sampler import BatchSampler, Sampler +from lightning.data.constants import _DEFAULT_CHUNK_BYTES, _TORCH_GREATER_EQUAL_2_1_0, _VIZ_TRACKER_AVAILABLE from lightning.data.streaming import Cache from lightning.data.streaming.combined import ( __NUM_SAMPLES_YIELDED_KEY__, __SAMPLES_KEY__, CombinedStreamingDataset, ) -from lightning.data.streaming.constants import _DEFAULT_CHUNK_BYTES, _TORCH_GREATER_EQUAL_2_1_0, _VIZ_TRACKER_AVAILABLE from lightning.data.streaming.dataset import StreamingDataset from lightning.data.streaming.sampler import CacheBatchSampler from lightning.data.utilities.env import _DistributedEnv diff --git a/src/lightning/data/streaming/dataset.py b/src/lightning/data/streaming/dataset.py index e281bbc0a1e2d..da0028184e123 100644 --- a/src/lightning/data/streaming/dataset.py +++ b/src/lightning/data/streaming/dataset.py @@ -19,11 +19,11 @@ import numpy as np from torch.utils.data import IterableDataset -from lightning.data.streaming import Cache -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _DEFAULT_CACHE_DIR, _INDEX_FILENAME, ) +from lightning.data.streaming import Cache from lightning.data.streaming.item_loader import BaseItemLoader from lightning.data.streaming.resolver import Dir, _resolve_dir from lightning.data.streaming.sampler import ChunkedIndex diff --git a/src/lightning/data/streaming/downloader.py b/src/lightning/data/streaming/downloader.py index 982b4d25142b3..03d7b9302068a 100644 --- a/src/lightning/data/streaming/downloader.py +++ b/src/lightning/data/streaming/downloader.py @@ -19,8 +19,8 @@ from filelock import FileLock, Timeout +from lightning.data.constants import _INDEX_FILENAME from lightning.data.streaming.client import S3Client -from lightning.data.streaming.constants import _INDEX_FILENAME class Downloader(ABC): diff --git a/src/lightning/data/streaming/item_loader.py b/src/lightning/data/streaming/item_loader.py index 779a683146182..2a1a02da67293 100644 --- a/src/lightning/data/streaming/item_loader.py +++ b/src/lightning/data/streaming/item_loader.py @@ -19,7 +19,7 @@ import numpy as np import torch -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _TORCH_DTYPES_MAPPING, _TORCH_GREATER_EQUAL_2_1_0, ) diff --git a/src/lightning/data/streaming/reader.py b/src/lightning/data/streaming/reader.py index 50705cb18f663..298452ea685f7 100644 --- a/src/lightning/data/streaming/reader.py +++ b/src/lightning/data/streaming/reader.py @@ -20,8 +20,8 @@ from threading import Thread from typing import Any, Dict, List, Optional, Tuple, Union +from lightning.data.constants import _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.config import ChunksConfig -from lightning.data.streaming.constants import _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.item_loader import BaseItemLoader, PyTreeLoader from lightning.data.streaming.sampler import ChunkedIndex from lightning.data.streaming.serializers import Serializer, _get_serializers diff --git a/src/lightning/data/streaming/serializers.py b/src/lightning/data/streaming/serializers.py index b689429953c21..9e5a92f520741 100644 --- a/src/lightning/data/streaming/serializers.py +++ b/src/lightning/data/streaming/serializers.py @@ -23,7 +23,7 @@ import torch from lightning_utilities.core.imports import RequirementCache -from lightning.data.streaming.constants import _NUMPY_DTYPES_MAPPING, _TORCH_DTYPES_MAPPING +from lightning.data.constants import _NUMPY_DTYPES_MAPPING, _TORCH_DTYPES_MAPPING _PIL_AVAILABLE = RequirementCache("PIL") _TORCH_VISION_AVAILABLE = RequirementCache("torchvision") diff --git a/src/lightning/data/streaming/writer.py b/src/lightning/data/streaming/writer.py index 44e6a8951773f..98b7a31e07d1b 100644 --- a/src/lightning/data/streaming/writer.py +++ b/src/lightning/data/streaming/writer.py @@ -21,8 +21,8 @@ import numpy as np import torch +from lightning.data.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.compression import _COMPRESSORS, Compressor -from lightning.data.streaming.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.serializers import Serializer, _get_serializers from lightning.data.utilities.env import _DistributedEnv, _WorkerEnv from lightning.data.utilities.format import _convert_bytes_to_int, _human_readable_bytes diff --git a/tests/tests_data/processing/test_dns.py b/tests/tests_data/processing/test_dns.py new file mode 100644 index 0000000000000..0703e346b02fd --- /dev/null +++ b/tests/tests_data/processing/test_dns.py @@ -0,0 +1,33 @@ +from unittest.mock import MagicMock + +from lightning.data.processing import dns as dns_module +from lightning.data.processing.dns import optimize_dns_context + + +def test_optimize_dns_context(monkeypatch): + popen_mock = MagicMock() + + monkeypatch.setattr(dns_module, "_IS_IN_STUDIO", True) + monkeypatch.setattr(dns_module, "Popen", popen_mock) + + class FakeFile: + + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + return self + + def readlines(self): + return ["127.0.0.53"] + + monkeypatch.setitem(__builtins__, "open", MagicMock(return_value=FakeFile())) + + with optimize_dns_context(True): + pass + + cmd = popen_mock._mock_call_args_list[0].args[0] + assert cmd == "sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing.dns import _optimize_dns; _optimize_dns(True)'" # noqa: E501 From baf8989ff9257b6eff500ae7aa369babfbea6d69 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Thu, 8 Feb 2024 12:31:52 +0100 Subject: [PATCH 3/8] Fix import error when running examples in fresh environment (#19431) (cherry picked from commit 5aea3b1fc3c538ad351d160b5cd2076f40194708) --- requirements/pytorch/examples.txt | 1 + src/lightning/pytorch/demos/transformer.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/requirements/pytorch/examples.txt b/requirements/pytorch/examples.txt index a1e0c0ab45e9e..a9dd656394c7c 100644 --- a/requirements/pytorch/examples.txt +++ b/requirements/pytorch/examples.txt @@ -1,6 +1,7 @@ # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment +requests <2.32.0 torchvision >=0.14.0, <0.17.0 gym[classic_control] >=0.17.0, <0.27.0 ipython[all] <8.15.0 diff --git a/src/lightning/pytorch/demos/transformer.py b/src/lightning/pytorch/demos/transformer.py index abb9e2e85a05c..4cd96115807ab 100644 --- a/src/lightning/pytorch/demos/transformer.py +++ b/src/lightning/pytorch/demos/transformer.py @@ -9,16 +9,19 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple -import requests import torch import torch.nn as nn import torch.nn.functional as F +from lightning_utilities.core.imports import RequirementCache from torch import Tensor from torch.nn.modules import MultiheadAttention from torch.utils.data import DataLoader, Dataset from lightning.pytorch import LightningModule +_REQUESTS_AVAILABLE = RequirementCache("requests") + + if hasattr(MultiheadAttention, "_reset_parameters") and not hasattr(MultiheadAttention, "reset_parameters"): # See https://github.com/pytorch/pytorch/issues/107909 MultiheadAttention.reset_parameters = MultiheadAttention._reset_parameters @@ -125,6 +128,11 @@ def __getitem__(self, index: int) -> Tuple[Tensor, Tensor]: @staticmethod def download(destination: Path) -> None: + if not _REQUESTS_AVAILABLE: + raise ModuleNotFoundError(str(_REQUESTS_AVAILABLE)) + + import requests + os.makedirs(destination.parent, exist_ok=True) url = "https://raw.githubusercontent.com/pytorch/examples/main/word_language_model/data/wikitext-2/train.txt" if os.path.exists(destination): From a90fec9fc5f89809727260f84f37e42d46751605 Mon Sep 17 00:00:00 2001 From: Xinyu Yang Date: Fri, 9 Feb 2024 23:21:28 +0800 Subject: [PATCH 4/8] bugfix: correct node rank (#19437) (cherry picked from commit 7b867c7d91ad8f154e9c5b4656271a6329a8b059) --- src/lightning/data/utilities/shuffle.py | 3 ++- tests/tests_data/{streaming => utilities}/test_shuffle.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) rename tests/tests_data/{streaming => utilities}/test_shuffle.py (91%) diff --git a/src/lightning/data/utilities/shuffle.py b/src/lightning/data/utilities/shuffle.py index 8be6563096ed6..4dc34a7dc2302 100644 --- a/src/lightning/data/utilities/shuffle.py +++ b/src/lightning/data/utilities/shuffle.py @@ -12,8 +12,9 @@ def _intra_node_chunk_shuffle( current_epoch: int, ) -> List[int]: chunk_indexes_per_nodes: Any = [[] for _ in range(distributed_env.num_nodes)] + process_per_node = distributed_env.world_size // distributed_env.num_nodes for rank, chunks_per_rank in enumerate(chunks_per_ranks): - chunk_indexes_per_nodes[0 if distributed_env.num_nodes == 1 else rank // distributed_env.num_nodes].extend( + chunk_indexes_per_nodes[0 if distributed_env.num_nodes == 1 else rank // process_per_node].extend( chunks_per_rank ) diff --git a/tests/tests_data/streaming/test_shuffle.py b/tests/tests_data/utilities/test_shuffle.py similarity index 91% rename from tests/tests_data/streaming/test_shuffle.py rename to tests/tests_data/utilities/test_shuffle.py index cb451ce73a1ec..9951a21c603ab 100644 --- a/tests/tests_data/streaming/test_shuffle.py +++ b/tests/tests_data/utilities/test_shuffle.py @@ -12,6 +12,11 @@ def test_intra_node_chunk_shuffle(): assert shuffled_indexes == [3, 2, 1, 0, 7, 6, 5, 4] + chunks_per_ranks = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13], [14, 15]] + shuffled_indexes = _intra_node_chunk_shuffle(_DistributedEnv(8, 7, 2), chunks_per_ranks, 42, 2) + assert shuffled_indexes == [5, 2, 0, 7, 6, 1, 3, 4, 13, 10, 8, 15, 14, 9, 11, 12] + + def test_associate_chunks_and_internals_to_ranks(): indexes = [0, 1, 2, 3, 4, 5, 6, 7] chunk_intervals = [[0, 50], [0, 50], [0, 50], [0, 50], [0, 50], [0, 50], [0, 50], [0, 50]] From a7660692e9a1b6b58cfdb141b8b6adfc1f16ce39 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 9 Feb 2024 16:54:54 +0100 Subject: [PATCH 5/8] Rename Lightning App CLI (#19440) (cherry picked from commit 0acd5f98102b69ee72878dbbd0953656fd10162b) --- .github/workflows/ci-pkg-install.yml | 2 +- docs/source-app/basics.rst | 6 +++--- .../go_beyond_training_content.rst | 6 +++--- .../jumpstart_from_app_gallery.rst | 4 ++-- .../get_started/training_with_apps.rst | 4 ++-- .../glossary/command_lines/command_lines.rst | 2 +- .../glossary/environment_variables.rst | 4 ++-- docs/source-app/glossary/secrets.rst | 6 +++--- .../glossary/use_local_lightning.rst | 2 +- docs/source-app/landing_app_run.bash | 2 +- .../hello_components/code_run_cloud.bash | 2 +- .../code_run_cloud_setup.bash | 2 +- .../hello_components/code_run_local.bash | 2 +- .../code_run_local_setup.bash | 2 +- .../workflows/add_server/any_server.rst | 4 ++-- .../workflows/add_web_ui/justpy/index.rst | 2 +- .../workflows/add_web_ui/panel/basic.rst | 6 +++--- .../react/connect_react_and_lightning.rst | 6 +++--- .../build_command_line_interface/cli.rst | 20 +++++++++---------- .../cli_client.rst | 18 ++++++++--------- .../build_command_line_interface/index.rst | 2 +- .../from_pytorch_lightning_script.rst | 8 ++++---- .../from_scratch_content.rst | 4 ++-- .../from_scratch_component_content.rst | 4 ++-- .../workflows/build_rest_api/add_api.rst | 2 +- .../run_app_on_cloud/lightning_cloud.rst | 4 ++-- docs/source-app/workflows/run_app_snippet.rst | 4 ++-- src/lightning/__setup__.py | 2 +- src/lightning/app/CHANGELOG.md | 6 ++++++ src/lightning/app/cli/app-template/README.md | 8 ++++---- src/lightning/app/cli/cmd_react_ui_init.py | 6 +++--- .../app/cli/commands/app_commands.py | 4 ++-- src/lightning/app/cli/commands/ls.py | 2 +- .../app/cli/component-template/README.md | 4 ++-- .../app/cli/component-template/setup.py | 2 +- src/lightning/app/cli/connect/app.py | 14 ++++++------- src/lightning/app/cli/lightning_cli.py | 2 +- .../app/cli/react-ui-template/README.md | 4 ++-- src/lightning/app/core/flow.py | 4 ++-- src/lightning/app/runners/cloud.py | 2 +- src/lightning/app/testing/testing.py | 2 +- src/lightning/app/utilities/load_app.py | 2 +- src/lightning/app/utilities/login.py | 2 +- src/lightning_app/__setup__.py | 2 +- tests/tests_app/cli/test_cli.py | 10 +++++----- tests/tests_app/cli/test_cmd_react_ui_init.py | 2 +- tests/tests_app/cli/test_connect.py | 8 +++++--- tests/tests_app/runners/test_cloud.py | 2 +- tests/tests_app/utilities/test_safe_pickle.py | 3 ++- 49 files changed, 116 insertions(+), 107 deletions(-) diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index c73d6adf5d3cd..60e1668e93e07 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -79,7 +79,7 @@ jobs: if: | ((matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app') || matrix.pkg-name == 'app' - run: lightning --version + run: lightning_app --version - name: DocTests actions working-directory: .actions/ diff --git a/docs/source-app/basics.rst b/docs/source-app/basics.rst index 8a64df8dcf62a..69b1bf751014b 100644 --- a/docs/source-app/basics.rst +++ b/docs/source-app/basics.rst @@ -85,7 +85,7 @@ And here's the output you get when running the above application using **Lightni .. code-block:: console - $ lightning run app docs/source/code_samples/quickstart/app_01.py + $ lightning_app run app docs/source/code_samples/quickstart/app_01.py INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view State: {'works': {'w_1': {'vars': {'counter': 1}}, 'w_2': {'vars': {'counter': 0}}}} @@ -175,7 +175,7 @@ Here's the output you get when running the above application using **Lightning C .. code-block:: console - $ lightning run app docs/source/code_samples/quickstart/app_02.py + $ lightning_app run app docs/source/code_samples/quickstart/app_02.py INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view # After you have clicked `run` on the UI. 0.0 0.0 @@ -230,7 +230,7 @@ When running the above app, we see the following logs: .. code-block:: console - $ lightning run app docs/source/code_samples/quickstart/app/app_0.py + $ lightning_app run app docs/source/code_samples/quickstart/app/app_0.py INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view # After you have clicked `run` on the UI. 0.0, 0.0 diff --git a/docs/source-app/get_started/go_beyond_training_content.rst b/docs/source-app/get_started/go_beyond_training_content.rst index f15831d28589c..5bc632b08ddd0 100644 --- a/docs/source-app/get_started/go_beyond_training_content.rst +++ b/docs/source-app/get_started/go_beyond_training_content.rst @@ -56,7 +56,7 @@ Install this App by typing: .. code-block:: bash - lightning install app lightning/quick-start + lightning_app install app lightning/quick-start Verify the App was successfully installed: @@ -74,7 +74,7 @@ Run the app locally with the ``run`` command 🤯 .. code:: bash - lightning run app app.py + lightning_app run app app.py ---- @@ -86,7 +86,7 @@ Add the ``--cloud`` argument to run on the `Lightning.AI cloud `_ script with Lightning Apps. diff --git a/docs/source-app/glossary/command_lines/command_lines.rst b/docs/source-app/glossary/command_lines/command_lines.rst index 693ddbeaba26f..15d77272c1692 100644 --- a/docs/source-app/glossary/command_lines/command_lines.rst +++ b/docs/source-app/glossary/command_lines/command_lines.rst @@ -16,7 +16,7 @@ A Command-line Interface (CLI) is an user interface (UI) in a terminal to intera .. note:: - The Lightning guideline to build CLI is `lightning ...` or ` ...`. + The Lightning guideline to build CLI is `lightning_app ...` or ` ...`. As an example, Lightning provides a CLI to interact with your Lightning Apps and the `lightning.ai `_ platform as follows: diff --git a/docs/source-app/glossary/environment_variables.rst b/docs/source-app/glossary/environment_variables.rst index 10c3e9ae2efac..c561a5dee4dfd 100644 --- a/docs/source-app/glossary/environment_variables.rst +++ b/docs/source-app/glossary/environment_variables.rst @@ -6,13 +6,13 @@ Environment Variables If your App is using configuration values you don't want to commit with your App source code, you can use environment variables. -Lightning allows you to set environment variables when running the App from the CLI with the `lightning run app` command. You can use environment variables to pass any values to the App, and avoiding sticking those values in the source code. +Lightning allows you to set environment variables when running the App from the CLI with the `lightning_app run app` command. You can use environment variables to pass any values to the App, and avoiding sticking those values in the source code. Set one or multiple variables using the **--env** option: .. code:: bash - lightning run app app.py --cloud --env FOO=BAR --env BAZ=FAZ + lightning_app run app app.py --cloud --env FOO=BAR --env BAZ=FAZ Environment variables are available in all Flows and Works, and can be accessed as follows: diff --git a/docs/source-app/glossary/secrets.rst b/docs/source-app/glossary/secrets.rst index 09a69ec2f6f92..0f1b226739af7 100644 --- a/docs/source-app/glossary/secrets.rst +++ b/docs/source-app/glossary/secrets.rst @@ -45,13 +45,13 @@ Use a secret .. code:: bash - lightning run app app.py --cloud --secret = + lightning_app run app app.py --cloud --secret = In this example, the command would be: .. code:: bash - lightning run app app.py --cloud --secret api_token=github_api_token + lightning_app run app app.py --cloud --secret api_token=github_api_token The ``--secret`` option can be used for multiple Secrets, and alongside the ``--env`` option. @@ -60,7 +60,7 @@ Here's an example: .. code:: bash - lightning run app app.py --cloud --env FOO=bar --secret MY_APP_SECRET=my-secret --secret ANOTHER_SECRET=another-secret + lightning_app run app app.py --cloud --env FOO=bar --secret MY_APP_SECRET=my-secret --secret ANOTHER_SECRET=another-secret ---- diff --git a/docs/source-app/glossary/use_local_lightning.rst b/docs/source-app/glossary/use_local_lightning.rst index dbc2f631a293c..1efc8e730e97a 100644 --- a/docs/source-app/glossary/use_local_lightning.rst +++ b/docs/source-app/glossary/use_local_lightning.rst @@ -9,7 +9,7 @@ git clone https://github.com/Lightning-AI/lightning.git cd lightning pip install -e . export PACKAGE_LIGHTNING=1 # <- this is the magic to use your version (not mainstream PyPI)! -lightning run app app.py --cloud +lightning_app run app app.py --cloud ``` By setting `PACKAGE_LIGHTNING=1`, lightning packages the lightning source code in your local directory in addition to your app source code and uploads them to the cloud. diff --git a/docs/source-app/landing_app_run.bash b/docs/source-app/landing_app_run.bash index fd306212e1e1c..e4bed41dadddd 100644 --- a/docs/source-app/landing_app_run.bash +++ b/docs/source-app/landing_app_run.bash @@ -2,4 +2,4 @@ pip install lightning # run the app on the --cloud (--setup installs deps automatically) -lightning run app app.py --setup --cloud +lightning_app run app app.py --setup --cloud diff --git a/docs/source-app/levels/basic/hello_components/code_run_cloud.bash b/docs/source-app/levels/basic/hello_components/code_run_cloud.bash index 6594fe0ecac33..f81431222724c 100644 --- a/docs/source-app/levels/basic/hello_components/code_run_cloud.bash +++ b/docs/source-app/levels/basic/hello_components/code_run_cloud.bash @@ -1 +1 @@ -lightning run app app.py --cloud +lightning_app run app app.py --cloud diff --git a/docs/source-app/levels/basic/hello_components/code_run_cloud_setup.bash b/docs/source-app/levels/basic/hello_components/code_run_cloud_setup.bash index ed69b303c279a..09435ff5d7caa 100644 --- a/docs/source-app/levels/basic/hello_components/code_run_cloud_setup.bash +++ b/docs/source-app/levels/basic/hello_components/code_run_cloud_setup.bash @@ -1 +1 @@ -lightning run app app.py --setup --cloud +lightning_app run app app.py --setup --cloud diff --git a/docs/source-app/levels/basic/hello_components/code_run_local.bash b/docs/source-app/levels/basic/hello_components/code_run_local.bash index 8a00b45e132ca..1f8c994a84b1d 100644 --- a/docs/source-app/levels/basic/hello_components/code_run_local.bash +++ b/docs/source-app/levels/basic/hello_components/code_run_local.bash @@ -1 +1 @@ -lightning run app app.py +lightning_app run app app.py diff --git a/docs/source-app/levels/basic/hello_components/code_run_local_setup.bash b/docs/source-app/levels/basic/hello_components/code_run_local_setup.bash index 11cc8a4c5a284..45b0529de3736 100644 --- a/docs/source-app/levels/basic/hello_components/code_run_local_setup.bash +++ b/docs/source-app/levels/basic/hello_components/code_run_local_setup.bash @@ -1 +1 @@ -lightning run app app.py --setup +lightning_app run app app.py --setup diff --git a/docs/source-app/workflows/add_server/any_server.rst b/docs/source-app/workflows/add_server/any_server.rst index 152303fdc2ee7..948ae22a2abc0 100644 --- a/docs/source-app/workflows/add_server/any_server.rst +++ b/docs/source-app/workflows/add_server/any_server.rst @@ -121,13 +121,13 @@ Start the app to see your new UI! .. code:: bash - lightning run app app.py + lightning_app run app app.py To run the app on the cloud, use the ``--cloud`` argument. .. code:: bash - lightning run app app.py --cloud + lightning_app run app app.py --cloud ---- diff --git a/docs/source-app/workflows/add_web_ui/justpy/index.rst b/docs/source-app/workflows/add_web_ui/justpy/index.rst index bf25a181dabf9..1626930c13d82 100644 --- a/docs/source-app/workflows/add_web_ui/justpy/index.rst +++ b/docs/source-app/workflows/add_web_ui/justpy/index.rst @@ -89,4 +89,4 @@ Now, you can run the Lightning App with: .. code-block:: - lightning run app app.py + lightning_app run app app.py diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 6d1d3bce8763d..b033312f6f0d1 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -111,7 +111,7 @@ Run the App locally: .. code:: bash - lightning run app app.py + lightning_app run app app.py The App should look like this: @@ -124,7 +124,7 @@ Now, run it on the cloud: .. code:: bash - lightning run app app.py --cloud + lightning_app run app app.py --cloud ---- @@ -319,7 +319,7 @@ Finally run the App .. code:: bash - lightning run app app.py + lightning_app run app app.py .. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-lightning-theme.gif :alt: Basic Panel Plotly Lightning App with theming diff --git a/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst b/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst index 034e031d5aa0c..bacdd3d299b05 100644 --- a/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst +++ b/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst @@ -13,11 +13,11 @@ Connect React to a Lightning app Example code ************ To illustrate how to connect a React app and a lightning App, we'll be using the `example_app.py` file -which :doc:`lightning init react-ui ` created: +which :doc:`lightning_app init react-ui ` created: .. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py -and the App.tsx file also created by :doc:`lightning init react-ui `: +and the App.tsx file also created by :doc:`lightning_app init react-ui `: .. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/ui/src/App.tsx @@ -54,7 +54,7 @@ At this point, the React app will render in the Lightning app. Test it out! .. code:: bash - lightning run app example_app.py + lightning_app run app example_app.py However, to make powerful React+Lightning apps, you must also connect the Lightning App state to the react app. These lines enable two-way communication between the react app and the Lightning app. diff --git a/docs/source-app/workflows/build_command_line_interface/cli.rst b/docs/source-app/workflows/build_command_line_interface/cli.rst index ffd9bc485fdd7..176e416b31630 100644 --- a/docs/source-app/workflows/build_command_line_interface/cli.rst +++ b/docs/source-app/workflows/build_command_line_interface/cli.rst @@ -30,7 +30,7 @@ Execute the following command in a terminal: .. code-block:: - lightning run app app.py + lightning_app run app app.py The following appears the terminal: @@ -47,19 +47,19 @@ The following appears the terminal: *************************** In another terminal, connect to the running App. -When you connect to an App, the Lightning CLI is replaced by the App CLI. To exit the App CLI, you need to run ``lightning disconnect``. +When you connect to an App, the Lightning CLI is replaced by the App CLI. To exit the App CLI, you need to run ``lightning_app disconnect``. .. code-block:: - lightning connect localhost + lightning_app connect localhost To see a list of available commands: .. code-block:: - lightning --help + lightning_app --help You are connected to the cloud Lightning App: localhost. - Usage: lightning [OPTIONS] COMMAND [ARGS]... + Usage: lightning_app [OPTIONS] COMMAND [ARGS]... --help Show this message and exit. @@ -70,9 +70,9 @@ To find the arguments of the commands: .. code-block:: - lightning add --help + lightning_app add --help You are connected to the cloud Lightning App: localhost. - Usage: lightning add [ARGS]... + Usage: lightning_app add [ARGS]... Options name: Add description @@ -87,7 +87,7 @@ Trigger the command line exposed by your App: .. code-block:: - lightning add --name=my_name + lightning_app add --name=my_name WARNING: Lightning Command Line Interface is an experimental feature and unannounced changes are likely. In your first terminal, **Received name: my_name** and **["my_name"]** are printed. @@ -106,11 +106,11 @@ In your first terminal, **Received name: my_name** and **["my_name"]** are print 5. Disconnect from the App ************************** -To exit the App CLI, you need to run ``lightning disconnect``. +To exit the App CLI, you need to run ``lightning_app disconnect``. .. code-block:: - lightning disconnect + lightning_app disconnect You are disconnected from the local Lightning App. ---- diff --git a/docs/source-app/workflows/build_command_line_interface/cli_client.rst b/docs/source-app/workflows/build_command_line_interface/cli_client.rst index f97062cdadbb6..589db930bf5fa 100644 --- a/docs/source-app/workflows/build_command_line_interface/cli_client.rst +++ b/docs/source-app/workflows/build_command_line_interface/cli_client.rst @@ -47,7 +47,7 @@ In a terminal, run the following command and open ``http://127.0.0.1:7501/docs`` .. code-block:: python - lightning run app app.py + lightning_app run app app.py Your Lightning App is starting. This won't take long. INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view @@ -58,11 +58,11 @@ In a terminal, run the following command and open ``http://127.0.0.1:7501/docs`` *************************** In another terminal, connect to the running App. -When you connect to an App, the Lightning CLI is replaced by the App CLI. To exit the App CLI, you need to run ``lightning disconnect``. +When you connect to an App, the Lightning CLI is replaced by the App CLI. To exit the App CLI, you need to run ``lightning_app disconnect``. .. code-block:: - lightning connect localhost + lightning_app connect localhost Storing `run_notebook` under /Users/thomas/.lightning/lightning_connection/commands/run_notebook.py You can review all the downloaded commands under /Users/thomas/.lightning/lightning_connection/commands folder. @@ -72,10 +72,10 @@ To see a list of available commands: .. code-block:: - lightning --help + lightning_app --help You are connected to the cloud Lightning App: localhost. - Usage: lightning [OPTIONS] COMMAND [ARGS]... + Usage: lightning_app [OPTIONS] COMMAND [ARGS]... --help Show this message and exit. @@ -87,7 +87,7 @@ To find the arguments of the commands: .. code-block:: - lightning run notebook --help + lightning_app run notebook --help You are connected to the cloud Lightning App: localhost. usage: notebook [-h] [--name NAME] [--cloud_compute CLOUD_COMPUTE] @@ -111,7 +111,7 @@ Run the first Notebook with the following command: .. code-block:: python - lightning run notebook --name="my_notebook" + lightning_app run notebook --name="my_notebook" WARNING: Lightning Command Line Interface is an experimental feature and unannounced changes are likely. The notebook my_notebook was created. @@ -119,7 +119,7 @@ And run a second notebook. .. code-block:: python - lightning run notebook --name="my_notebook_2" + lightning_app run notebook --name="my_notebook_2" WARNING: Lightning Command Line Interface is an experimental feature and unannounced changes are likely. The notebook my_notebook_2 was created. @@ -141,7 +141,7 @@ To exit the App CLI, you need to run **lightning disconnect**. .. code-block:: - lightning disconnect + lightning_app disconnect You are disconnected from the local Lightning App. ---- diff --git a/docs/source-app/workflows/build_command_line_interface/index.rst b/docs/source-app/workflows/build_command_line_interface/index.rst index ada49b45c51ff..9a3e24f784910 100644 --- a/docs/source-app/workflows/build_command_line_interface/index.rst +++ b/docs/source-app/workflows/build_command_line_interface/index.rst @@ -14,7 +14,7 @@ A Command-line Interface (CLI) is an user interface (UI) in a terminal to intera .. note:: - The Lightning guideline to build CLI is `lightning ...` or ` ...`. + The Lightning guideline to build CLI is `lightning_app ...` or ` ...`. As an example, Lightning provides a CLI to interact with your Lightning Apps and the `lightning.ai `_ platform as follows: diff --git a/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst b/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst index 1a09083434ba0..f489edaedc13f 100644 --- a/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst +++ b/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst @@ -26,14 +26,14 @@ To develop a template from a PyTorch Lightning script, use this command: .. code:: bash - lightning init pl-app path/to/the/pl_script.py + lightning_app init pl-app path/to/the/pl_script.py If your script is not at the root of the project folder, and you'd like to include all source files within that folder, you can specify the root path as the first argument: .. code:: bash - lightning init pl-app path/to/project/root path/to/the/pl_script.py + lightning_app init pl-app path/to/project/root path/to/the/pl_script.py The default trainer App lets you train a model with a beautiful UI locally and on the cloud with zero effort! @@ -50,13 +50,13 @@ Run the App locally: .. code:: bash - lightning run app pl-app/app.py + lightning_app run app pl-app/app.py Or run the App on the cloud so you can share with collaborators and even use all the cloud GPUs you want. .. code:: bash - lightning run app pl-app/app.py --cloud + lightning_app run app pl-app/app.py --cloud .. figure:: https://storage.googleapis.com/grid-packages/pytorch-lightning-app/docs-thumbnail.png diff --git a/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst b/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst index 7641b4f4e7c30..a528fa6088736 100644 --- a/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst +++ b/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst @@ -51,10 +51,10 @@ Run the Lightning App locally: .. code:: bash - lightning run app app.py + lightning_app run app app.py Run the Lightning App on the cloud: .. code:: bash - lightning run app app.py --cloud + lightning_app run app app.py --cloud diff --git a/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst b/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst index b5828e6952cc6..b29c19db40129 100644 --- a/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst +++ b/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst @@ -91,7 +91,7 @@ run the app .. code:: bash - lightning run app app.py + lightning_app run app app.py ---- @@ -150,4 +150,4 @@ run the app .. code:: bash - lightning run app app.py + lightning_app run app app.py diff --git a/docs/source-app/workflows/build_rest_api/add_api.rst b/docs/source-app/workflows/build_rest_api/add_api.rst index 5ed8b4fc4ca82..00fd16d18715a 100644 --- a/docs/source-app/workflows/build_rest_api/add_api.rst +++ b/docs/source-app/workflows/build_rest_api/add_api.rst @@ -28,7 +28,7 @@ Execute the following command in a terminal: .. code-block:: - lightning run app app.py + lightning_app run app app.py The following appears: diff --git a/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst b/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst index 83418f23a0b84..494a523852bd9 100644 --- a/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst +++ b/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst @@ -13,7 +13,7 @@ To run any app on the public lightning cloud use the ``--cloud`` argument: .. code:: bash - lightning run app app.py --cloud + lightning_app run app app.py --cloud .. note:: @@ -44,7 +44,7 @@ Simply use the ``--name`` flag when running your app, for example: .. code:: bash - lightning run app app.py --cloud --name my-awesome-app + lightning_app run app app.py --cloud --name my-awesome-app Alternatively, you can change the name of the app in the ``.lightning`` file: diff --git a/docs/source-app/workflows/run_app_snippet.rst b/docs/source-app/workflows/run_app_snippet.rst index 57d2ae095d449..6283cb88104eb 100644 --- a/docs/source-app/workflows/run_app_snippet.rst +++ b/docs/source-app/workflows/run_app_snippet.rst @@ -13,7 +13,7 @@ Run the app with the ``run`` command .. code:: bash - lightning run app app.py + lightning_app run app app.py .. raw:: html @@ -25,7 +25,7 @@ Add the ``--cloud`` argument to run on the `lightning cloud Dict[str, Any]: "python_requires": ">=3.8", # todo: take the lowes based on all packages "entry_points": { "console_scripts": [ - "lightning = lightning:_cli_entry_point", + "lightning_app = lightning:_cli_entry_point", ], }, "setup_requires": [], diff --git a/src/lightning/app/CHANGELOG.md b/src/lightning/app/CHANGELOG.md index 46e37963fa9e5..d09c302b22067 100644 --- a/src/lightning/app/CHANGELOG.md +++ b/src/lightning/app/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [2.2.0] - 2024-02-09 + +## Changed + +- Renames the `lightning` cli to `lightning_app` ([#19440](https://github.com/Lightning-AI/pytorch-lightning/pull/19440)) + ## [2.1.4] - 2024-01-31 diff --git a/src/lightning/app/cli/app-template/README.md b/src/lightning/app/cli/app-template/README.md index f5b1d4c17c212..76c88e6cedb38 100644 --- a/src/lightning/app/cli/app-template/README.md +++ b/src/lightning/app/cli/app-template/README.md @@ -3,7 +3,7 @@ This ⚡ [Lightning app](https://lightning.ai/) ⚡ was generated automatically with: ```bash -lightning init app placeholdername +lightning_app init app placeholdername ``` ## To run placeholdername @@ -11,19 +11,19 @@ lightning init app placeholdername First, install placeholdername (warning: this app has not been officially approved on the lightning gallery): ```bash -lightning install app https://github.com/theUser/placeholdername +lightning_app install app https://github.com/theUser/placeholdername ``` Once the app is installed, run it locally with: ```bash -lightning run app placeholdername/app.py +lightning_app run app placeholdername/app.py ``` Run it on the [lightning cloud](https://lightning.ai/) with: ```bash -lightning run app placeholdername/app.py --cloud +lightning_app run app placeholdername/app.py --cloud ``` ## to test and link diff --git a/src/lightning/app/cli/cmd_react_ui_init.py b/src/lightning/app/cli/cmd_react_ui_init.py index 9dcbfe61b8161..22e668433e233 100644 --- a/src/lightning/app/cli/cmd_react_ui_init.py +++ b/src/lightning/app/cli/cmd_react_ui_init.py @@ -59,7 +59,7 @@ def configure_layout(self): return la.frontend.StaticWebFrontend(Path(__file__).parent / "react-ui/src/dist") ⚡ run the example_app.py to see it live! - lightning run app {dest_dir}/example_app.py + lightning_app run app {dest_dir}/example_app.py """ logger.info(m) @@ -80,7 +80,7 @@ def _check_react_prerequisites() -> None: if not has_npm: m = """ - This machine is missing 'npm'. Please install npm and rerun 'lightning init react-ui' again. + This machine is missing 'npm'. Please install npm and rerun 'lightning_app init react-ui' again. Install instructions: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm """ @@ -94,7 +94,7 @@ def _check_react_prerequisites() -> None: if not has_node: m = """ - This machine is missing 'node'. Please install node and rerun 'lightning init react-ui' again. + This machine is missing 'node'. Please install node and rerun 'lightning_app init react-ui' again. Install instructions: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm """ diff --git a/src/lightning/app/cli/commands/app_commands.py b/src/lightning/app/cli/commands/app_commands.py index 2251ef74f981a..bbecceabb6e28 100644 --- a/src/lightning/app/cli/commands/app_commands.py +++ b/src/lightning/app/cli/commands/app_commands.py @@ -61,7 +61,7 @@ def _run_app_command(app_name: str, app_id: Optional[str]): if value == command and "_" in value: print( f"The command `{value}` was provided with an underscore and it isn't allowed." - f"Instead, use `lightning {value.replace('_', ' ')}`." + f"Instead, use `lightning_app {value.replace('_', ' ')}`." ) sys.exit(0) break @@ -88,7 +88,7 @@ def _run_app_command(app_name: str, app_id: Optional[str]): def _handle_command_without_client(command: str, metadata: Dict, url: str) -> None: supported_params = list(metadata["parameters"]) if _is_running_help(sys.argv): - print(f"Usage: lightning {command} [ARGS]...") + print(f"Usage: lightning_app {command} [ARGS]...") print(" ") print("Options") for param in supported_params: diff --git a/src/lightning/app/cli/commands/ls.py b/src/lightning/app/cli/commands/ls.py index 8ba0077aa2c98..e16a354f66d8c 100644 --- a/src/lightning/app/cli/commands/ls.py +++ b/src/lightning/app/cli/commands/ls.py @@ -81,7 +81,7 @@ def ls(path: Optional[str] = None, print: bool = True, use_live: bool = True) -> # This happens if the user changes cluster and the project doesn't exit. if len(project) == 0: return _error_and_exit( - f"There isn't any Lightning Project matching the name {splits[0]}." " HINT: Use `lightning cd`." + f"There isn't any Lightning Project matching the name {splits[0]}." " HINT: Use `lightning_app cd`." ) project_id = project[0].project_id diff --git a/src/lightning/app/cli/component-template/README.md b/src/lightning/app/cli/component-template/README.md index 733e415e11e0a..1d700e286461b 100644 --- a/src/lightning/app/cli/component-template/README.md +++ b/src/lightning/app/cli/component-template/README.md @@ -3,7 +3,7 @@ This ⚡ [Lightning component](https://lightning.ai/) ⚡ was generated automatically with: ```bash -lightning init component placeholdername +lightning_app init component placeholdername ``` ## To run placeholdername @@ -11,7 +11,7 @@ lightning init component placeholdername First, install placeholdername (warning: this component has not been officially approved on the lightning gallery): ```bash -lightning install component https://github.com/theUser/placeholdername +lightning_app install component https://github.com/theUser/placeholdername ``` Once the app is installed, use it in an app: diff --git a/src/lightning/app/cli/component-template/setup.py b/src/lightning/app/cli/component-template/setup.py index f5b9e87c32f67..78631901190b2 100644 --- a/src/lightning/app/cli/component-template/setup.py +++ b/src/lightning/app/cli/component-template/setup.py @@ -5,7 +5,7 @@ setup( name="placeholdername", version="0.0.0", - description="⚡ Lightning component ⚡ generated with command: lightning init component", + description="⚡ Lightning component ⚡ generated with command: lightning_app init component", author="", author_email="", # REPLACE WITH YOUR OWN GITHUB PROJECT LINK diff --git a/src/lightning/app/cli/connect/app.py b/src/lightning/app/cli/connect/app.py index c2e2adc385a54..e3f0d0b151bb5 100644 --- a/src/lightning/app/cli/connect/app.py +++ b/src/lightning/app/cli/connect/app.py @@ -129,11 +129,11 @@ def connect_app(app_name_or_id: str): with open(connected_file, "w") as f: f.write(app_name_or_id + "\n") - click.echo("The lightning CLI now responds to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning App CLI now responds to app commands. Use 'lightning_app --help' to see them.") click.echo(" ") Popen( - f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", + f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning_app --help", shell=True, stdout=sys.stdout, stderr=sys.stderr, @@ -147,11 +147,11 @@ def connect_app(app_name_or_id: str): shutil.copytree(matched_commands, commands) shutil.copy(matched_connected_file, connected_file) - click.echo("The lightning CLI now responds to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning App CLI now responds to app commands. Use 'lightning_app --help' to see them.") click.echo(" ") Popen( - f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", + f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning_app --help", shell=True, stdout=sys.stdout, stderr=sys.stderr, @@ -205,11 +205,11 @@ def connect_app(app_name_or_id: str): f.write(retriever.app_name + "\n") f.write(retriever.app_id + "\n") - click.echo("The lightning CLI now responds to app commands. Use 'lightning --help' to see them.") + click.echo("The lightning App CLI now responds to app commands. Use 'lightning_app --help' to see them.") click.echo(" ") Popen( - f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning --help", + f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning_app --help", shell=True, stdout=sys.stdout, stderr=sys.stderr, @@ -238,7 +238,7 @@ def disconnect_app(logout: bool = False): if not logout: click.echo( "You aren't connected to any Lightning App. " - "Please use `lightning connect app_name_or_id` to connect to one." + "Please use `lightning_app connect app_name_or_id` to connect to one." ) diff --git a/src/lightning/app/cli/lightning_cli.py b/src/lightning/app/cli/lightning_cli.py index 9bf8877fac975..6ef78fbc1d83b 100644 --- a/src/lightning/app/cli/lightning_cli.py +++ b/src/lightning/app/cli/lightning_cli.py @@ -104,7 +104,7 @@ def main() -> None: _run_app_command(app_name, app_id) click.echo() - click.echo(message + " Return to the primary CLI with `lightning disconnect`.") + click.echo(message + " Return to the primary CLI with `lightning_app disconnect`.") else: _main() diff --git a/src/lightning/app/cli/react-ui-template/README.md b/src/lightning/app/cli/react-ui-template/README.md index 023fb3babc14e..1f28215d22199 100644 --- a/src/lightning/app/cli/react-ui-template/README.md +++ b/src/lightning/app/cli/react-ui-template/README.md @@ -5,7 +5,7 @@ This is a full react template ready to use in a component This UI was automatically generated with: ```commandline -lightning init react-ui +lightning_app init react-ui ``` ### Delete files @@ -26,7 +26,7 @@ This template comes with `example_app.py` to show how to integrate the UI into a run it with: ```bash -lightning run app react-ui/example_app.py +lightning_app run app react-ui/example_app.py ``` ### Connect React to your component diff --git a/src/lightning/app/core/flow.py b/src/lightning/app/core/flow.py index f8321e691d472..f9ffcca61c5a9 100644 --- a/src/lightning/app/core/flow.py +++ b/src/lightning/app/core/flow.py @@ -716,11 +716,11 @@ def my_remote_method(self, name): .. code-block:: bash - lightning run app app.py + lightning_app run app app.py .. code-block:: bash - lightning my_command_name --args name=my_own_name + lightning_app my_command_name --args name=my_own_name """ raise NotImplementedError diff --git a/src/lightning/app/runners/cloud.py b/src/lightning/app/runners/cloud.py index 1eaa2fa14f99d..f0d0fe224669b 100644 --- a/src/lightning/app/runners/cloud.py +++ b/src/lightning/app/runners/cloud.py @@ -131,7 +131,7 @@ def open(self, name: str, cluster_id: Optional[str] = None): user = self.backend.client.auth_service_get_user() if not user.features.code_tab: rich.print( - "[red]The `lightning open` command has not been enabled for your account. " + "[red]The `lightning_app open` command has not been enabled for your account. " "To request access, please contact support@lightning.ai[/red]" ) sys.exit(1) diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py index eda33c6f5d6a5..456c645d49e7b 100644 --- a/src/lightning/app/testing/testing.py +++ b/src/lightning/app/testing/testing.py @@ -275,7 +275,7 @@ def run_app_in_cloud( token = res.json()["token"] # 3. Disconnect from the App if any. - Popen("lightning logout", shell=True).wait() + Popen("lightning_app logout", shell=True).wait() # 4. Launch the application in the cloud from the Lightning CLI. with tempfile.TemporaryDirectory() as tmpdir: diff --git a/src/lightning/app/utilities/load_app.py b/src/lightning/app/utilities/load_app.py index f5c5fdb40cb8d..b2016194bfe57 100644 --- a/src/lightning/app/utilities/load_app.py +++ b/src/lightning/app/utilities/load_app.py @@ -239,7 +239,7 @@ def _patch_sys_argv(): """This function modifies the ``sys.argv`` by extracting the arguments after ``--app_args`` and removed everything else before executing the user app script. - The command: ``lightning run app app.py --without-server --app_args --use_gpu --env ...`` will be converted into + The command: ``lightning_app run app app.py --without-server --app_args --use_gpu --env ...`` will be converted into ``app.py --use_gpu`` """ diff --git a/src/lightning/app/utilities/login.py b/src/lightning/app/utilities/login.py index dbab4af090183..b725a39acb736 100644 --- a/src/lightning/app/utilities/login.py +++ b/src/lightning/app/utilities/login.py @@ -144,7 +144,7 @@ def authenticate(self) -> Optional[str]: raise ValueError( "We couldn't find any credentials linked to your account. " - "Please try logging in using the CLI command `lightning login`" + "Please try logging in using the CLI command `lightning_app login`" ) diff --git a/src/lightning_app/__setup__.py b/src/lightning_app/__setup__.py index 69bb9750c1cd9..080eb6c3a6399 100644 --- a/src/lightning_app/__setup__.py +++ b/src/lightning_app/__setup__.py @@ -85,7 +85,7 @@ def _setup_args() -> Dict[str, Any]: "python_requires": ">=3.8", "entry_points": { "console_scripts": [ - "lightning = lightning_app.cli.lightning_cli:main", + "lightning_app = lightning_app.cli.lightning_cli:main", ], }, "setup_requires": [], diff --git a/tests/tests_app/cli/test_cli.py b/tests/tests_app/cli/test_cli.py index 6bdc957b7ea3f..cfc747d729919 100644 --- a/tests/tests_app/cli/test_cli.py +++ b/tests/tests_app/cli/test_cli.py @@ -20,7 +20,7 @@ def test_commands(command): def test_main_lightning_cli_no_arguments(): """Validate the Lightning CLI without args.""" - res = os.popen("lightning").read() + res = os.popen("lightning_app").read() assert "login " in res assert "logout " in res assert "run " in res @@ -31,7 +31,7 @@ def test_main_lightning_cli_no_arguments(): def test_main_lightning_cli_help(): """Validate the Lightning CLI.""" - res = os.popen("lightning --help").read() + res = os.popen("lightning_app --help").read() assert "login " in res assert "logout " in res assert "run " in res @@ -39,7 +39,7 @@ def test_main_lightning_cli_help(): assert "delete " in res assert "show " in res - res = os.popen("lightning run --help").read() + res = os.popen("lightning_app run --help").read() assert "app " in res # hidden run commands should not appear in the help text @@ -49,7 +49,7 @@ def test_main_lightning_cli_help(): assert "frontend" not in res # inspect show group - res = os.popen("lightning show --help").read() + res = os.popen("lightning_app show --help").read() assert "logs " in res @@ -86,7 +86,7 @@ def test_cli_logout(exists: mock.MagicMock, unlink: mock.MagicMock, creds: bool) def test_lightning_cli_version(): - res = os.popen("lightning --version").read() + res = os.popen("lightning_app --version").read() assert __version__ in res diff --git a/tests/tests_app/cli/test_cmd_react_ui_init.py b/tests/tests_app/cli/test_cmd_react_ui_init.py index 2016f78615e3d..0f28420e291ba 100644 --- a/tests/tests_app/cli/test_cmd_react_ui_init.py +++ b/tests/tests_app/cli/test_cmd_react_ui_init.py @@ -30,7 +30,7 @@ def test_missing_yarn(): @_RunIf(skip_windows=True) def test_copy_and_setup_react_ui(tmpdir): dest_dir = os.path.join(tmpdir, "react-ui") - os.system(f"lightning init react-ui --dest_dir={dest_dir}") + os.system(f"lightning_app init react-ui --dest_dir={dest_dir}") # make sure package is minimal files = sorted(f for f in os.listdir(dest_dir) if f != "__pycache__") diff --git a/tests/tests_app/cli/test_connect.py b/tests/tests_app/cli/test_connect.py index b29c358e95154..960f466eed655 100644 --- a/tests/tests_app/cli/test_connect.py +++ b/tests/tests_app/cli/test_connect.py @@ -69,7 +69,8 @@ def fn(msg): messages = [] disconnect_app() assert messages == [ - "You aren't connected to any Lightning App. Please use `lightning connect app_name_or_id` to connect to one." + "You aren't connected to any Lightning App." + " Please use `lightning_app connect app_name_or_id` to connect to one." ] assert _retrieve_connection_to_an_app() == (None, None) @@ -147,7 +148,7 @@ def fn(msg): messages = [] connect_app("example") - assert "The lightning CLI now responds to app commands" in messages[0] + assert "The lightning App CLI now responds to app commands" in messages[0] messages = [] disconnect_app() @@ -162,7 +163,8 @@ def fn(msg): messages = [] disconnect_app() assert messages == [ - "You aren't connected to any Lightning App. Please use `lightning connect app_name_or_id` to connect to one." + "You aren't connected to any Lightning App." + " Please use `lightning_app connect app_name_or_id` to connect to one." ] assert _retrieve_connection_to_an_app() == (None, None) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index 971d12b701577..821049d308180 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -1565,7 +1565,7 @@ def test_not_enabled(self, monkeypatch, capsys): out, _ = capsys.readouterr() assert exited - assert "`lightning open` command has not been enabled" in out + assert "`lightning_app open` command has not been enabled" in out class TestCloudspaceDispatch: diff --git a/tests/tests_app/utilities/test_safe_pickle.py b/tests/tests_app/utilities/test_safe_pickle.py index 473fe28ed22f7..989804af2c79d 100644 --- a/tests/tests_app/utilities/test_safe_pickle.py +++ b/tests/tests_app/utilities/test_safe_pickle.py @@ -5,7 +5,8 @@ def test_safe_pickle_app(): test_dir = Path(__file__).parent / "testdata" proc = subprocess.Popen( - ["lightning", "run", "app", "safe_pickle_app.py", "--open-ui", "false"], stdout=subprocess.PIPE, cwd=test_dir + ["lightning_app", "run", "app", "safe_pickle_app.py", "--open-ui", "false"], + stdout=subprocess.PIPE, cwd=test_dir, ) stdout, _ = proc.communicate() assert "Exiting the pickling app successfully" in stdout.decode("UTF-8") From f9c06b940d7f3d457e2d77efa3e887f2da1c23a1 Mon Sep 17 00:00:00 2001 From: Xinyu Yang Date: Sat, 10 Feb 2024 01:08:28 +0800 Subject: [PATCH 6/8] bugfix: skip write index.json if no data is wrote. (#19439) (cherry picked from commit 47c8f4cba089a78fa3fe31dcac6a43416bc13820) --- src/lightning/data/streaming/writer.py | 3 ++- tests/tests_data/processing/test_data_processor.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lightning/data/streaming/writer.py b/src/lightning/data/streaming/writer.py index 98b7a31e07d1b..db20d93460a2b 100644 --- a/src/lightning/data/streaming/writer.py +++ b/src/lightning/data/streaming/writer.py @@ -322,6 +322,8 @@ def write_chunk_to_file( def write_chunks_index(self) -> str: """Write the chunks index to a JSON file.""" + if len(self._chunks_info) == 0: + return "" filepath = os.path.join(self._cache_dir, f"{self.rank}.{_INDEX_FILENAME}") config = self.get_config() with open(filepath, "w") as out: @@ -393,7 +395,6 @@ def _merge_no_wait(self, node_rank: Optional[int] = None) -> None: config = data["config"] elif config != data["config"]: - breakpoint() raise Exception("The config isn't consistent between chunks. This shouldn't have happened.") chunks_info.extend(data["chunks"]) diff --git a/tests/tests_data/processing/test_data_processor.py b/tests/tests_data/processing/test_data_processor.py index 06420130321a9..797d4911d1282 100644 --- a/tests/tests_data/processing/test_data_processor.py +++ b/tests/tests_data/processing/test_data_processor.py @@ -564,6 +564,9 @@ def test_data_processsor_nlp(tmpdir, monkeypatch): data_processor = DataProcessor(input_dir=str(tmpdir), num_workers=1, num_downloaders=1) data_processor.run(TextTokenizeRecipe(chunk_size=1024 * 11)) + data_processor_more_wokers = DataProcessor(input_dir=str(tmpdir), num_workers=2, num_downloaders=1) + data_processor_more_wokers.run(TextTokenizeRecipe(chunk_size=1024 * 11)) + class ImageResizeRecipe(DataTransformRecipe): def prepare_structure(self, input_dir: str): From 609a9c8753ecd78f27dc772bfe2fd954e006ce08 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Mon, 12 Feb 2024 17:22:53 +0100 Subject: [PATCH 7/8] Rename Lightning Fabric CLI (#19442) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> (cherry picked from commit 2ed7282f7c21f03aa5b61a99c55cd7e7f90aab29) --- docs/source-fabric/fundamentals/launch.rst | 12 ++++----- src/lightning/__setup__.py | 2 ++ src/lightning/fabric/CHANGELOG.md | 1 + src/lightning/fabric/cli.py | 30 +++++++++++++++++++++- src/lightning_fabric/__setup__.py | 5 ++++ tests/tests_fabric/test_cli.py | 22 ++++++++++------ 6 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/source-fabric/fundamentals/launch.rst b/docs/source-fabric/fundamentals/launch.rst index 0f25ebcda6021..d30a75621300c 100644 --- a/docs/source-fabric/fundamentals/launch.rst +++ b/docs/source-fabric/fundamentals/launch.rst @@ -67,7 +67,7 @@ An alternative way to launch your Python script in multiple processes is to use .. code-block:: bash - lightning run model path/to/your/script.py + fabric run model path/to/your/script.py This is essentially the same as running ``python path/to/your/script.py``, but it also lets you configure the following settings externally without changing your code: @@ -80,9 +80,9 @@ This is essentially the same as running ``python path/to/your/script.py``, but i .. code-block:: bash - lightning run model --help + fabric run model --help - Usage: lightning run model [OPTIONS] SCRIPT [SCRIPT_ARGS]... + Usage: fabric run model [OPTIONS] SCRIPT [SCRIPT_ARGS]... Run a Lightning Fabric script. @@ -128,7 +128,7 @@ Here is how you run DDP with 8 GPUs and `torch.bfloat16 `_ w .. code-block:: bash - lightning run model ./path/to/train.py \ + fabric run model ./path/to/train.py \ --strategy=deepspeed_stage_3 \ --devices=8 \ --accelerator=cuda \ @@ -148,7 +148,7 @@ Or `DeepSpeed Zero3 `_ w .. code-block:: bash - lightning run model ./path/to/train.py \ + fabric run model ./path/to/train.py \ --devices=auto \ --accelerator=auto \ --precision=16 diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index b7e57309eb674..b9b19143bbe13 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -114,6 +114,8 @@ def _setup_args() -> Dict[str, Any]: "python_requires": ">=3.8", # todo: take the lowes based on all packages "entry_points": { "console_scripts": [ + "fabric = lightning.fabric.cli:_main", + "lightning = lightning.fabric.cli:_legacy_main", "lightning_app = lightning:_cli_entry_point", ], }, diff --git a/src/lightning/fabric/CHANGELOG.md b/src/lightning/fabric/CHANGELOG.md index ab2e568b7db61..95332ac22cb29 100644 --- a/src/lightning/fabric/CHANGELOG.md +++ b/src/lightning/fabric/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Changed +- Rename `lightning run model` to `fabric run model` ([#19442](https://github.com/Lightning-AI/pytorch-lightning/pull/19442)) - `seed_everything()` without passing in a seed no longer randomly selects a seed, and now defaults to `0` ([#18846](https://github.com/Lightning-AI/lightning/pull/18846)) - Changed the `TransformerEnginePrecision(dtype=)` argument to `weights_dtype` and made it required ([#19082](https://github.com/Lightning-AI/lightning/pull/19082)) - The columns in the `metrics.csv` file produced by `CSVLogger` are now sorted alphabetically ([#19159](https://github.com/Lightning-AI/lightning/pull/19159)) diff --git a/src/lightning/fabric/cli.py b/src/lightning/fabric/cli.py index 0ff777232a7fb..2805675cecf70 100644 --- a/src/lightning/fabric/cli.py +++ b/src/lightning/fabric/cli.py @@ -14,6 +14,8 @@ import logging import os import re +import subprocess +import sys from argparse import Namespace from typing import Any, List, Optional @@ -29,6 +31,7 @@ _log = logging.getLogger(__name__) _CLICK_AVAILABLE = RequirementCache("click") +_LIGHTNING_SDK_AVAILABLE = RequirementCache("lightning_sdk") _SUPPORTED_ACCELERATORS = ("cpu", "gpu", "cuda", "mps", "tpu") @@ -44,7 +47,32 @@ def _get_supported_strategies() -> List[str]: if _CLICK_AVAILABLE: import click - @click.command( + def _legacy_main() -> None: + """Legacy CLI handler for fabric. + + Raises deprecation warning and runs through fabric cli if necessary, else runs the entrypoint directly + + """ + print("`lightning run model` is deprecated and will be removed in future versions." + " Please call `fabric run model` instead.") + args = sys.argv[1:] + if args and args[0] == "run" and args[1] == "model": + _main() + return + + if _LIGHTNING_SDK_AVAILABLE: + subprocess.run([sys.executable, "-m", "lightning_sdk.cli.entrypoint"] + args) + return + + @click.group() + def _main() -> None: + pass + + @_main.group() + def run() -> None: + pass + + @run.command( "model", context_settings={ "ignore_unknown_options": True, diff --git a/src/lightning_fabric/__setup__.py b/src/lightning_fabric/__setup__.py index 869ffad571f7e..8fe0bc0937ef5 100644 --- a/src/lightning_fabric/__setup__.py +++ b/src/lightning_fabric/__setup__.py @@ -78,6 +78,11 @@ def _setup_args() -> Dict[str, Any]: "install_requires": assistant.load_requirements( _PATH_REQUIREMENTS, unfreeze="none" if _FREEZE_REQUIREMENTS else "all" ), + "entry_points": { + "console_scripts": [ + "fabric = lightning_fabric.cli:_main", + ], + }, "extras_require": _prepare_extras(), "project_urls": { "Bug Tracker": "https://github.com/Lightning-AI/lightning/issues", diff --git a/tests/tests_fabric/test_cli.py b/tests/tests_fabric/test_cli.py index c8e9a6bbedd57..596318b4b619f 100644 --- a/tests/tests_fabric/test_cli.py +++ b/tests/tests_fabric/test_cli.py @@ -21,7 +21,6 @@ import pytest from lightning.fabric.cli import _get_supported_strategies, _run_model -from lightning_utilities.core.imports import ModuleAvailableCache from tests_fabric.helpers.runif import RunIf @@ -176,13 +175,20 @@ def test_cli_torchrun_num_processes_launched(_, devices, expected, monkeypatch, ) +def test_cli_through_fabric_entry_point(): + result = subprocess.run("fabric run model --help", capture_output=True, text=True, shell=True) + + message = "Usage: fabric run model [OPTIONS] SCRIPT [SCRIPT_ARGS]" + assert message in result.stdout or message in result.stderr + @pytest.mark.skipif("lightning.fabric" == "lightning_fabric", reason="standalone package") def test_cli_through_lightning_entry_point(): result = subprocess.run("lightning run model --help", capture_output=True, text=True, shell=True) - if not ModuleAvailableCache("lightning.app"): - message = "The `lightning` command requires additional dependencies" - assert message in result.stdout or message in result.stderr - assert result.returncode != 0 - else: - message = "Usage: lightning run model [OPTIONS] SCRIPT [SCRIPT_ARGS]" - assert message in result.stdout or message in result.stderr + + deprecation_message = ( + "`lightning run model` is deprecated and will be removed in future versions. " + "Please call `fabric run model` instead" + ) + message = "Usage: lightning run model [OPTIONS] SCRIPT [SCRIPT_ARGS]" + assert deprecation_message in result.stdout + assert message in result.stdout or message in result.stderr From 6e86937d33247ea688d5d949e2bb69d4de9a0ced Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 12 Feb 2024 17:27:03 +0100 Subject: [PATCH 8/8] releasing `2.2.0.post0` --- src/version.info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.info b/src/version.info index ccbccc3dc6263..a8d39b5507816 100644 --- a/src/version.info +++ b/src/version.info @@ -1 +1 @@ -2.2.0 +2.2.0.post0