diff --git a/docs/source-pytorch/api_references.rst b/docs/source-pytorch/api_references.rst index ba95e74428a156..96a061a941b57d 100644 --- a/docs/source-pytorch/api_references.rst +++ b/docs/source-pytorch/api_references.rst @@ -12,7 +12,7 @@ accelerators Accelerator CPUAccelerator - GPUAccelerator + CUDAAccelerator HPUAccelerator IPUAccelerator TPUAccelerator diff --git a/docs/source-pytorch/common/trainer.rst b/docs/source-pytorch/common/trainer.rst index ee4cd6c5c0005a..b7c4c21018dcc5 100644 --- a/docs/source-pytorch/common/trainer.rst +++ b/docs/source-pytorch/common/trainer.rst @@ -249,8 +249,8 @@ Example:: .. code-block:: python - # This is part of the built-in `GPUAccelerator` - class GPUAccelerator(Accelerator): + # This is part of the built-in `CUDAAccelerator` + class CUDAAccelerator(Accelerator): """Accelerator for GPU devices.""" @staticmethod @@ -603,8 +603,8 @@ based on the accelerator type (``"cpu", "gpu", "tpu", "ipu", "auto"``). .. code-block:: python - # This is part of the built-in `GPUAccelerator` - class GPUAccelerator(Accelerator): + # This is part of the built-in `CUDAAccelerator` + class CUDAAccelerator(Accelerator): """Accelerator for GPU devices.""" @staticmethod diff --git a/docs/source-pytorch/extensions/accelerator.rst b/docs/source-pytorch/extensions/accelerator.rst index fdfe9660b90aad..5fedd441fdd2c8 100644 --- a/docs/source-pytorch/extensions/accelerator.rst +++ b/docs/source-pytorch/extensions/accelerator.rst @@ -125,7 +125,7 @@ Accelerator API Accelerator CPUAccelerator - GPUAccelerator + CUDAAccelerator HPUAccelerator IPUAccelerator MPSAccelerator diff --git a/src/pytorch_lightning/accelerators/__init__.py b/src/pytorch_lightning/accelerators/__init__.py index e7d757cd731492..b4521d931c734f 100644 --- a/src/pytorch_lightning/accelerators/__init__.py +++ b/src/pytorch_lightning/accelerators/__init__.py @@ -12,7 +12,7 @@ # limitations under the License. from pytorch_lightning.accelerators.accelerator import Accelerator # noqa: F401 from pytorch_lightning.accelerators.cpu import CPUAccelerator # noqa: F401 -from pytorch_lightning.accelerators.gpu import GPUAccelerator # noqa: F401 +from pytorch_lightning.accelerators.cuda import CUDAAccelerator # noqa: F401 from pytorch_lightning.accelerators.hpu import HPUAccelerator # noqa: F401 from pytorch_lightning.accelerators.ipu import IPUAccelerator # noqa: F401 from pytorch_lightning.accelerators.mps import MPSAccelerator # noqa: F401 diff --git a/src/pytorch_lightning/accelerators/gpu.py b/src/pytorch_lightning/accelerators/cuda.py similarity index 94% rename from src/pytorch_lightning/accelerators/gpu.py rename to src/pytorch_lightning/accelerators/cuda.py index 898ce09b914310..89d1a5b284b0c3 100644 --- a/src/pytorch_lightning/accelerators/gpu.py +++ b/src/pytorch_lightning/accelerators/cuda.py @@ -28,8 +28,8 @@ _log = logging.getLogger(__name__) -class GPUAccelerator(Accelerator): - """Accelerator for GPU devices.""" +class CUDAAccelerator(Accelerator): + """Accelerator for NVIDIA CUDA devices.""" def setup_environment(self, root_device: torch.device) -> None: """ @@ -92,6 +92,12 @@ def is_available() -> bool: @classmethod def register_accelerators(cls, accelerator_registry: Dict) -> None: + accelerator_registry.register( + "cuda", + cls, + description=f"{cls.__class__.__name__}", + ) + # temporarily enable "gpu" to point to the CUDA Accelerator accelerator_registry.register( "gpu", cls, diff --git a/src/pytorch_lightning/loops/dataloader/evaluation_loop.py b/src/pytorch_lightning/loops/dataloader/evaluation_loop.py index baf65d566d2dc8..3a9c9ec0ac3919 100644 --- a/src/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/src/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -23,7 +23,7 @@ from torch.utils.data.dataloader import DataLoader import pytorch_lightning as pl -from pytorch_lightning.accelerators import GPUAccelerator +from pytorch_lightning.accelerators import CUDAAccelerator from pytorch_lightning.callbacks.progress.rich_progress import _RICH_AVAILABLE from pytorch_lightning.loops.dataloader import DataLoaderLoop from pytorch_lightning.loops.epoch import EvaluationEpochLoop @@ -411,7 +411,7 @@ def _select_data_fetcher_type(trainer: "pl.Trainer") -> Type[AbstractDataFetcher ) return DataLoaderIterDataFetcher elif os.getenv("PL_INTER_BATCH_PARALLELISM", "0") == "1": - if not isinstance(trainer.accelerator, GPUAccelerator): + if not isinstance(trainer.accelerator, CUDAAccelerator): raise MisconfigurationException("Inter batch parallelism is available only when using Nvidia GPUs.") return InterBatchParallelDataFetcher return DataFetcher diff --git a/src/pytorch_lightning/loops/fit_loop.py b/src/pytorch_lightning/loops/fit_loop.py index ab63b0e6df3be3..8b54579a6bbfb9 100644 --- a/src/pytorch_lightning/loops/fit_loop.py +++ b/src/pytorch_lightning/loops/fit_loop.py @@ -17,7 +17,7 @@ from typing import Optional, Type import pytorch_lightning as pl -from pytorch_lightning.accelerators import GPUAccelerator +from pytorch_lightning.accelerators import CUDAAccelerator from pytorch_lightning.loops import Loop from pytorch_lightning.loops.epoch import TrainingEpochLoop from pytorch_lightning.loops.epoch.training_epoch_loop import _OUTPUTS_TYPE as _EPOCH_OUTPUTS_TYPE @@ -340,7 +340,7 @@ def _select_data_fetcher(trainer: "pl.Trainer") -> Type[AbstractDataFetcher]: ) return DataLoaderIterDataFetcher elif os.getenv("PL_INTER_BATCH_PARALLELISM", "0") == "1": - if not isinstance(trainer.accelerator, GPUAccelerator): + if not isinstance(trainer.accelerator, CUDAAccelerator): raise MisconfigurationException("Inter batch parallelism is available only when using Nvidia GPUs.") return InterBatchParallelDataFetcher return DataFetcher diff --git a/src/pytorch_lightning/strategies/deepspeed.py b/src/pytorch_lightning/strategies/deepspeed.py index 9b4d3513c1aaba..ede42754aafc9d 100644 --- a/src/pytorch_lightning/strategies/deepspeed.py +++ b/src/pytorch_lightning/strategies/deepspeed.py @@ -27,7 +27,7 @@ from torch.optim import Optimizer import pytorch_lightning as pl -from pytorch_lightning.accelerators.gpu import GPUAccelerator +from pytorch_lightning.accelerators.cuda import CUDAAccelerator from pytorch_lightning.core.optimizer import _init_optimizers_and_lr_schedulers from pytorch_lightning.overrides.base import _LightningModuleWrapperBase, _LightningPrecisionModuleWrapperBase from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment @@ -452,7 +452,7 @@ def init_deepspeed(self): if self.lightning_module.trainer.gradient_clip_algorithm == GradClipAlgorithmType.VALUE: raise MisconfigurationException("DeepSpeed does not support clipping gradients by value.") - if not isinstance(self.accelerator, GPUAccelerator): + if not isinstance(self.accelerator, CUDAAccelerator): raise MisconfigurationException( f"DeepSpeed strategy is only supported on GPU but `{self.accelerator.__class__.__name__}` is used." ) diff --git a/src/pytorch_lightning/strategies/hivemind.py b/src/pytorch_lightning/strategies/hivemind.py index b274856bb6113c..b258fe7f738adf 100644 --- a/src/pytorch_lightning/strategies/hivemind.py +++ b/src/pytorch_lightning/strategies/hivemind.py @@ -172,9 +172,9 @@ def num_peers(self) -> int: @property def root_device(self) -> torch.device: from pytorch_lightning.accelerators.cpu import CPUAccelerator - from pytorch_lightning.accelerators.gpu import GPUAccelerator + from pytorch_lightning.accelerators.cuda import CUDAAccelerator - if isinstance(self.accelerator, GPUAccelerator): + if isinstance(self.accelerator, CUDAAccelerator): return torch.device(f"cuda:{torch.cuda.current_device()}") elif isinstance(self.accelerator, CPUAccelerator): return torch.device("cpu") diff --git a/src/pytorch_lightning/trainer/connectors/accelerator_connector.py b/src/pytorch_lightning/trainer/connectors/accelerator_connector.py index 2e112c754cbc5d..ece0e5d27bdceb 100644 --- a/src/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/src/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -22,7 +22,7 @@ from pytorch_lightning.accelerators.accelerator import Accelerator from pytorch_lightning.accelerators.cpu import CPUAccelerator -from pytorch_lightning.accelerators.gpu import GPUAccelerator +from pytorch_lightning.accelerators.cuda import CUDAAccelerator from pytorch_lightning.accelerators.hpu import HPUAccelerator from pytorch_lightning.accelerators.ipu import IPUAccelerator from pytorch_lightning.accelerators.mps import MPSAccelerator @@ -370,12 +370,12 @@ def _check_config_and_set_final_flags( ) self._accelerator_flag = "cpu" if self._strategy_flag.parallel_devices[0].type == "cuda": - if self._accelerator_flag and self._accelerator_flag not in ("auto", "gpu"): + if self._accelerator_flag and self._accelerator_flag not in ("auto", "cuda", "gpu"): raise MisconfigurationException( f"GPU parallel_devices set through {self._strategy_flag.__class__.__name__} class," f" but accelerator set to {self._accelerator_flag}, please choose one device type" ) - self._accelerator_flag = "gpu" + self._accelerator_flag = "cuda" self._parallel_devices = self._strategy_flag.parallel_devices amp_type = amp_type if isinstance(amp_type, str) else None @@ -475,7 +475,7 @@ def _map_deprecated_devices_specific_info_to_accelerator_and_device_flag( if tpu_cores: self._accelerator_flag = "tpu" if gpus: - self._accelerator_flag = "gpu" + self._accelerator_flag = "cuda" if num_processes: self._accelerator_flag = "cpu" @@ -497,7 +497,7 @@ def _choose_accelerator(self) -> str: if MPSAccelerator.is_available(): return "mps" if torch.cuda.is_available() and torch.cuda.device_count() > 0: - return "gpu" + return "cuda" return "cpu" def _set_parallel_devices_and_init_accelerator(self) -> None: @@ -534,7 +534,7 @@ def _set_devices_flag_if_auto_passed(self) -> None: self._devices_flag = self.accelerator.auto_device_count() def _set_devices_flag_if_auto_select_gpus_passed(self) -> None: - if self._auto_select_gpus and isinstance(self._gpus, int) and isinstance(self.accelerator, GPUAccelerator): + if self._auto_select_gpus and isinstance(self._gpus, int) and isinstance(self.accelerator, CUDAAccelerator): self._devices_flag = pick_multiple_gpus(self._gpus) log.info(f"Auto select gpus: {self._devices_flag}") @@ -579,8 +579,8 @@ def _choose_strategy(self) -> Union[Strategy, str]: return DDPStrategy.strategy_name if len(self._parallel_devices) <= 1: # TODO: Change this once gpu accelerator was renamed to cuda accelerator - if isinstance(self._accelerator_flag, (GPUAccelerator, MPSAccelerator)) or ( - isinstance(self._accelerator_flag, str) and self._accelerator_flag in ("gpu", "mps") + if isinstance(self._accelerator_flag, (CUDAAccelerator, MPSAccelerator)) or ( + isinstance(self._accelerator_flag, str) and self._accelerator_flag in ("cuda", "gpu", "mps") ): device = device_parser.determine_root_gpu_device(self._parallel_devices) else: @@ -609,7 +609,7 @@ def _check_strategy_and_fallback(self) -> None: if ( strategy_flag in DDPFullyShardedNativeStrategy.get_registered_strategies() or isinstance(self._strategy_flag, DDPFullyShardedNativeStrategy) - ) and self._accelerator_flag != "gpu": + ) and self._accelerator_flag not in ("cuda", "gpu"): raise MisconfigurationException( f"You selected strategy to be `{DDPFullyShardedNativeStrategy.strategy_name}`, " "but GPU accelerator is not used." @@ -632,7 +632,7 @@ def _handle_horovod(self) -> None: ) hvd.init() - if isinstance(self.accelerator, GPUAccelerator): + if isinstance(self.accelerator, CUDAAccelerator): # Horovod assigns one local GPU per process self._parallel_devices = [torch.device(f"cuda:{i}") for i in range(hvd.local_size())] else: diff --git a/src/pytorch_lightning/trainer/trainer.py b/src/pytorch_lightning/trainer/trainer.py index 25357578ea24e2..882326f870de6f 100644 --- a/src/pytorch_lightning/trainer/trainer.py +++ b/src/pytorch_lightning/trainer/trainer.py @@ -38,7 +38,7 @@ import pytorch_lightning as pl from pytorch_lightning.accelerators import ( Accelerator, - GPUAccelerator, + CUDAAccelerator, HPUAccelerator, IPUAccelerator, MPSAccelerator, @@ -1735,7 +1735,7 @@ def __setup_profiler(self) -> None: def _log_device_info(self) -> None: - if GPUAccelerator.is_available(): + if CUDAAccelerator.is_available(): gpu_available = True gpu_type = " (cuda)" elif MPSAccelerator.is_available(): @@ -1745,7 +1745,7 @@ def _log_device_info(self) -> None: gpu_available = False gpu_type = "" - gpu_used = isinstance(self.accelerator, (GPUAccelerator, MPSAccelerator)) + gpu_used = isinstance(self.accelerator, (CUDAAccelerator, MPSAccelerator)) rank_zero_info(f"GPU available: {gpu_available}{gpu_type}, used: {gpu_used}") num_tpu_cores = self.num_devices if isinstance(self.accelerator, TPUAccelerator) else 0 @@ -1758,10 +1758,10 @@ def _log_device_info(self) -> None: rank_zero_info(f"HPU available: {_HPU_AVAILABLE}, using: {num_hpus} HPUs") # TODO: Integrate MPS Accelerator here, once gpu maps to both - if torch.cuda.is_available() and not isinstance(self.accelerator, GPUAccelerator): + if torch.cuda.is_available() and not isinstance(self.accelerator, CUDAAccelerator): rank_zero_warn( "GPU available but not used. Set `accelerator` and `devices` using" - f" `Trainer(accelerator='gpu', devices={GPUAccelerator.auto_device_count()})`.", + f" `Trainer(accelerator='gpu', devices={CUDAAccelerator.auto_device_count()})`.", category=PossibleUserWarning, ) @@ -2069,7 +2069,7 @@ def root_gpu(self) -> Optional[int]: "`Trainer.root_gpu` is deprecated in v1.6 and will be removed in v1.8. " "Please use `Trainer.strategy.root_device.index` instead." ) - return self.strategy.root_device.index if isinstance(self.accelerator, GPUAccelerator) else None + return self.strategy.root_device.index if isinstance(self.accelerator, CUDAAccelerator) else None @property def tpu_cores(self) -> int: @@ -2093,7 +2093,7 @@ def num_gpus(self) -> int: "`Trainer.num_gpus` was deprecated in v1.6 and will be removed in v1.8." " Please use `Trainer.num_devices` instead." ) - return self.num_devices if isinstance(self.accelerator, GPUAccelerator) else 0 + return self.num_devices if isinstance(self.accelerator, CUDAAccelerator) else 0 @property def devices(self) -> int: @@ -2109,7 +2109,7 @@ def data_parallel_device_ids(self) -> Optional[List[int]]: "`Trainer.data_parallel_device_ids` was deprecated in v1.6 and will be removed in v1.8." " Please use `Trainer.device_ids` instead." ) - return self.device_ids if isinstance(self.accelerator, GPUAccelerator) else None + return self.device_ids if isinstance(self.accelerator, CUDAAccelerator) else None @property def lightning_module(self) -> "pl.LightningModule": diff --git a/src/pytorch_lightning/utilities/memory.py b/src/pytorch_lightning/utilities/memory.py index 286a571001b0fe..573dd6ed0f129c 100644 --- a/src/pytorch_lightning/utilities/memory.py +++ b/src/pytorch_lightning/utilities/memory.py @@ -101,7 +101,7 @@ def get_gpu_memory_map() -> Dict[str, float]: r""" .. deprecated:: v1.5 This function was deprecated in v1.5 in favor of - `pytorch_lightning.accelerators.gpu._get_nvidia_gpu_stats` and will be removed in v1.7. + `pytorch_lightning.accelerators.cuda._get_nvidia_gpu_stats` and will be removed in v1.7. Get the current gpu usage. diff --git a/tests/tests_pytorch/accelerators/test_accelerator_connector.py b/tests/tests_pytorch/accelerators/test_accelerator_connector.py index 100a4cc1d1c7a1..33911bffb0eb75 100644 --- a/tests/tests_pytorch/accelerators/test_accelerator_connector.py +++ b/tests/tests_pytorch/accelerators/test_accelerator_connector.py @@ -24,7 +24,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators.accelerator import Accelerator from pytorch_lightning.accelerators.cpu import CPUAccelerator -from pytorch_lightning.accelerators.gpu import GPUAccelerator +from pytorch_lightning.accelerators.cuda import CUDAAccelerator from pytorch_lightning.accelerators.mps import MPSAccelerator from pytorch_lightning.plugins import DoublePrecisionPlugin, LayerSync, NativeSyncBatchNorm, PrecisionPlugin from pytorch_lightning.plugins.environments import ( @@ -259,14 +259,14 @@ def test_accelerator_cpu(_): with pytest.raises( MisconfigurationException, - match="GPUAccelerator can not run on your system since the accelerator is not available.", + match="CUDAAccelerator can not run on your system since the accelerator is not available.", ): with pytest.deprecated_call(match=r"is deprecated in v1.7 and will be removed"): Trainer(gpus=1) with pytest.raises( MisconfigurationException, - match="GPUAccelerator can not run on your system since the accelerator is not available.", + match="CUDAAccelerator can not run on your system since the accelerator is not available.", ): Trainer(accelerator="gpu") @@ -287,13 +287,13 @@ def test_accelererator_invalid_type_devices(mock_is_available, mock_device_count @RunIf(min_cuda_gpus=1) def test_accelerator_gpu(): trainer = Trainer(accelerator="gpu", devices=1) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) trainer = Trainer(accelerator="gpu") - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) trainer = Trainer(accelerator="auto", devices=1) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) @pytest.mark.parametrize(["devices", "strategy_class"], [(1, SingleDeviceStrategy), (5, DDPSpawnStrategy)]) @@ -312,13 +312,13 @@ def test_accelerator_gpu_with_devices(devices, strategy_class): trainer = Trainer(accelerator="gpu", devices=devices) assert trainer.num_devices == len(devices) if isinstance(devices, list) else devices assert isinstance(trainer.strategy, strategy_class) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) @RunIf(min_cuda_gpus=1) def test_accelerator_auto_with_devices_gpu(): trainer = Trainer(accelerator="auto", devices=1) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert trainer.num_devices == 1 @@ -392,7 +392,7 @@ def test_device_type_when_strategy_instance_gpu_passed(strategy_class): trainer = Trainer(strategy=strategy_class(), accelerator="gpu", devices=2) assert isinstance(trainer.strategy, strategy_class) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) @pytest.mark.parametrize("precision", [1, 12, "invalid"]) @@ -419,7 +419,7 @@ def test_strategy_choice_ddp_spawn_cpu(): @mock.patch("torch.cuda.is_available", return_value=True) def test_strategy_choice_ddp(*_): trainer = Trainer(fast_dev_run=True, strategy="ddp", accelerator="gpu", devices=1) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert isinstance(trainer.strategy, DDPStrategy) assert isinstance(trainer.strategy.cluster_environment, LightningEnvironment) @@ -429,7 +429,7 @@ def test_strategy_choice_ddp(*_): @mock.patch("torch.cuda.is_available", return_value=True) def test_strategy_choice_ddp_spawn(cuda_available_mock, device_count_mock): trainer = Trainer(fast_dev_run=True, strategy="ddp_spawn", accelerator="gpu", devices=1) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert isinstance(trainer.strategy, DDPSpawnStrategy) assert isinstance(trainer.strategy.cluster_environment, LightningEnvironment) @@ -451,7 +451,7 @@ def test_strategy_choice_ddp_spawn(cuda_available_mock, device_count_mock): def test_strategy_choice_ddp_slurm(setup_distributed_mock, strategy): trainer = Trainer(fast_dev_run=True, strategy=strategy, accelerator="gpu", devices=2) assert trainer._accelerator_connector._is_slurm_managing_tasks() - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert isinstance(trainer.strategy, DDPStrategy) assert isinstance(trainer.strategy.cluster_environment, SLURMEnvironment) assert trainer.strategy.cluster_environment.local_rank() == 1 @@ -477,7 +477,7 @@ def test_strategy_choice_ddp_slurm(setup_distributed_mock, strategy): @mock.patch("torch.cuda.is_available", return_value=True) def test_strategy_choice_ddp_te(*_): trainer = Trainer(fast_dev_run=True, strategy="ddp", accelerator="gpu", devices=2) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert isinstance(trainer.strategy, DDPStrategy) assert isinstance(trainer.strategy.cluster_environment, TorchElasticEnvironment) assert trainer.strategy.cluster_environment.local_rank() == 1 @@ -524,7 +524,7 @@ def test_strategy_choice_ddp_cpu_te(*_): @mock.patch("torch.cuda.is_available", return_value=True) def test_strategy_choice_ddp_kubeflow(*_): trainer = Trainer(fast_dev_run=True, strategy="ddp", accelerator="gpu", devices=1) - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert isinstance(trainer.strategy, DDPStrategy) assert isinstance(trainer.strategy.cluster_environment, KubeflowEnvironment) assert trainer.strategy.cluster_environment.local_rank() == 0 @@ -649,7 +649,7 @@ def test_devices_auto_choice_cpu( def test_devices_auto_choice_gpu(is_gpu_available_mock, device_count_mock): trainer = Trainer(accelerator="auto", devices="auto") - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert trainer.num_devices == 2 diff --git a/tests/tests_pytorch/accelerators/test_accelerator_registry.py b/tests/tests_pytorch/accelerators/test_accelerator_registry.py index 11c806a21c740d..791d4c33dbbe84 100644 --- a/tests/tests_pytorch/accelerators/test_accelerator_registry.py +++ b/tests/tests_pytorch/accelerators/test_accelerator_registry.py @@ -63,4 +63,4 @@ def is_available(): def test_available_accelerators_in_registry(): - assert AcceleratorRegistry.available_accelerators() == ["cpu", "gpu", "hpu", "ipu", "mps", "tpu"] + assert AcceleratorRegistry.available_accelerators() == ["cpu", "cuda", "gpu", "hpu", "ipu", "mps", "tpu"] diff --git a/tests/tests_pytorch/accelerators/test_common.py b/tests/tests_pytorch/accelerators/test_common.py index 3de26b58883906..9395c7e84c709b 100644 --- a/tests/tests_pytorch/accelerators/test_common.py +++ b/tests/tests_pytorch/accelerators/test_common.py @@ -14,14 +14,14 @@ from unittest import mock from pytorch_lightning import Trainer -from pytorch_lightning.accelerators import Accelerator, CPUAccelerator, GPUAccelerator, IPUAccelerator, TPUAccelerator +from pytorch_lightning.accelerators import Accelerator, CPUAccelerator, CUDAAccelerator, IPUAccelerator, TPUAccelerator from pytorch_lightning.strategies import DDPStrategy @mock.patch("torch.cuda.device_count", return_value=2) def test_auto_device_count(device_count_mock): assert CPUAccelerator.auto_device_count() == 1 - assert GPUAccelerator.auto_device_count() == 2 + assert CUDAAccelerator.auto_device_count() == 2 assert TPUAccelerator.auto_device_count() == 8 assert IPUAccelerator.auto_device_count() == 4 diff --git a/tests/tests_pytorch/accelerators/test_gpu.py b/tests/tests_pytorch/accelerators/test_gpu.py index f6334780d75a51..e660ff270f9214 100644 --- a/tests/tests_pytorch/accelerators/test_gpu.py +++ b/tests/tests_pytorch/accelerators/test_gpu.py @@ -17,8 +17,8 @@ import torch from pytorch_lightning import Trainer -from pytorch_lightning.accelerators import GPUAccelerator -from pytorch_lightning.accelerators.gpu import get_nvidia_gpu_stats +from pytorch_lightning.accelerators import CUDAAccelerator +from pytorch_lightning.accelerators.cuda import get_nvidia_gpu_stats from pytorch_lightning.demos.boring_classes import BoringModel from tests_pytorch.helpers.runif import RunIf @@ -26,7 +26,7 @@ @RunIf(min_cuda_gpus=1) def test_get_torch_gpu_stats(tmpdir): current_device = torch.device(f"cuda:{torch.cuda.current_device()}") - gpu_stats = GPUAccelerator().get_device_stats(current_device) + gpu_stats = CUDAAccelerator().get_device_stats(current_device) fields = ["allocated_bytes.all.freed", "inactive_split.all.peak", "reserved_bytes.large_pool.peak"] for f in fields: @@ -62,7 +62,7 @@ def test_set_cuda_device(set_device_mock, tmpdir): @RunIf(min_cuda_gpus=1) def test_gpu_availability(): - assert GPUAccelerator.is_available() + assert CUDAAccelerator.is_available() @RunIf(min_cuda_gpus=1) diff --git a/tests/tests_pytorch/callbacks/test_quantization.py b/tests/tests_pytorch/callbacks/test_quantization.py index 1a12728c7facea..41d0810a0aec89 100644 --- a/tests/tests_pytorch/callbacks/test_quantization.py +++ b/tests/tests_pytorch/callbacks/test_quantization.py @@ -20,7 +20,7 @@ from torchmetrics.functional import mean_absolute_percentage_error as mape from pytorch_lightning import seed_everything, Trainer -from pytorch_lightning.accelerators import GPUAccelerator +from pytorch_lightning.accelerators import CUDAAccelerator from pytorch_lightning.callbacks import QuantizationAwareTraining from pytorch_lightning.demos.boring_classes import RandomDataset from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -38,9 +38,9 @@ @RunIf(quantization=True, max_torch="1.11") def test_quantization(tmpdir, observe: str, fuse: bool, convert: bool): """Parity test for quant model.""" - cuda_available = GPUAccelerator.is_available() + cuda_available = CUDAAccelerator.is_available() - if observe == "average" and not fuse and GPUAccelerator.is_available(): + if observe == "average" and not fuse and CUDAAccelerator.is_available(): pytest.xfail("TODO: flakiness in GPU CI") seed_everything(42) diff --git a/tests/tests_pytorch/models/test_gpu.py b/tests/tests_pytorch/models/test_gpu.py index ffd093e6ee0e3f..bdc61ca399e127 100644 --- a/tests/tests_pytorch/models/test_gpu.py +++ b/tests/tests_pytorch/models/test_gpu.py @@ -23,7 +23,7 @@ import tests_pytorch.helpers.pipelines as tpipes import tests_pytorch.helpers.utils as tutils from pytorch_lightning import Trainer -from pytorch_lightning.accelerators import CPUAccelerator, GPUAccelerator +from pytorch_lightning.accelerators import CPUAccelerator, CUDAAccelerator from pytorch_lightning.demos.boring_classes import BoringModel from pytorch_lightning.plugins.environments import TorchElasticEnvironment from pytorch_lightning.utilities import device_parser @@ -196,7 +196,7 @@ def test_torchelastic_gpu_parsing(mocked_device_count, mocked_is_available, gpus assert isinstance(trainer._accelerator_connector.cluster_environment, TorchElasticEnvironment) # when use gpu if device_parser.parse_gpu_ids(gpus, include_cuda=True) is not None: - assert isinstance(trainer.accelerator, GPUAccelerator) + assert isinstance(trainer.accelerator, CUDAAccelerator) assert trainer.num_devices == len(gpus) if isinstance(gpus, list) else gpus assert trainer.device_ids == device_parser.parse_gpu_ids(gpus, include_cuda=True) # fall back to cpu diff --git a/tests/tests_pytorch/plugins/test_cluster_integration.py b/tests/tests_pytorch/plugins/test_cluster_integration.py index a5c4f7e101761b..c413b0015db618 100644 --- a/tests/tests_pytorch/plugins/test_cluster_integration.py +++ b/tests/tests_pytorch/plugins/test_cluster_integration.py @@ -59,7 +59,7 @@ def environment_combinations(): "strategy_cls", [DDPStrategy, DDPShardedStrategy, pytest.param(DeepSpeedStrategy, marks=RunIf(deepspeed=True))], ) -@mock.patch("pytorch_lightning.accelerators.gpu.GPUAccelerator.is_available", return_value=True) +@mock.patch("pytorch_lightning.accelerators.cuda.CUDAAccelerator.is_available", return_value=True) def test_ranks_available_manual_strategy_selection(mock_gpu_acc_available, strategy_cls): """Test that the rank information is readily available after Trainer initialization.""" num_nodes = 2 diff --git a/tests/tests_pytorch/strategies/test_ddp.py b/tests/tests_pytorch/strategies/test_ddp.py index 58fa28559b97f8..003fe2250b5752 100644 --- a/tests/tests_pytorch/strategies/test_ddp.py +++ b/tests/tests_pytorch/strategies/test_ddp.py @@ -103,7 +103,7 @@ def test_torch_distributed_backend_env_variables(tmpdir): @mock.patch("torch.cuda.set_device") @mock.patch("torch.cuda.is_available", return_value=True) @mock.patch("torch.cuda.device_count", return_value=1) -@mock.patch("pytorch_lightning.accelerators.gpu.GPUAccelerator.is_available", return_value=True) +@mock.patch("pytorch_lightning.accelerators.cuda.CUDAAccelerator.is_available", return_value=True) @mock.patch.dict(os.environ, {"PL_TORCH_DISTRIBUTED_BACKEND": "gloo"}, clear=True) def test_ddp_torch_dist_is_available_in_setup( mock_gpu_is_available, mock_device_count, mock_cuda_available, mock_set_device, tmpdir diff --git a/tests/tests_pytorch/trainer/test_trainer.py b/tests/tests_pytorch/trainer/test_trainer.py index a0d20fc58ed1c4..c46c0168db5581 100644 --- a/tests/tests_pytorch/trainer/test_trainer.py +++ b/tests/tests_pytorch/trainer/test_trainer.py @@ -34,7 +34,7 @@ import pytorch_lightning import tests_pytorch.helpers.utils as tutils from pytorch_lightning import Callback, LightningDataModule, LightningModule, Trainer -from pytorch_lightning.accelerators import CPUAccelerator, GPUAccelerator +from pytorch_lightning.accelerators import CPUAccelerator, CUDAAccelerator from pytorch_lightning.callbacks import EarlyStopping, GradientAccumulationScheduler, ModelCheckpoint, Timer from pytorch_lightning.callbacks.fault_tolerance import _FaultToleranceCheckpoint from pytorch_lightning.callbacks.prediction_writer import BasePredictionWriter @@ -1967,21 +1967,21 @@ def training_step(self, batch, batch_idx): {"strategy": None, "accelerator": "gpu", "devices": 1}, SingleDeviceStrategy, "single_device", - GPUAccelerator, + CUDAAccelerator, 1, ), - ({"strategy": "dp", "accelerator": "gpu", "devices": 1}, DataParallelStrategy, "dp", GPUAccelerator, 1), - ({"strategy": "ddp", "accelerator": "gpu", "devices": 1}, DDPStrategy, "ddp", GPUAccelerator, 1), + ({"strategy": "dp", "accelerator": "gpu", "devices": 1}, DataParallelStrategy, "dp", CUDAAccelerator, 1), + ({"strategy": "ddp", "accelerator": "gpu", "devices": 1}, DDPStrategy, "ddp", CUDAAccelerator, 1), ( {"strategy": "ddp_spawn", "accelerator": "gpu", "devices": 1}, DDPSpawnStrategy, "ddp_spawn", - GPUAccelerator, + CUDAAccelerator, 1, ), - ({"strategy": None, "accelerator": "gpu", "devices": 2}, DDPSpawnStrategy, "ddp_spawn", GPUAccelerator, 2), - ({"strategy": "dp", "accelerator": "gpu", "devices": 2}, DataParallelStrategy, "dp", GPUAccelerator, 2), - ({"strategy": "ddp", "accelerator": "gpu", "devices": 2}, DDPStrategy, "ddp", GPUAccelerator, 2), + ({"strategy": None, "accelerator": "gpu", "devices": 2}, DDPSpawnStrategy, "ddp_spawn", CUDAAccelerator, 2), + ({"strategy": "dp", "accelerator": "gpu", "devices": 2}, DataParallelStrategy, "dp", CUDAAccelerator, 2), + ({"strategy": "ddp", "accelerator": "gpu", "devices": 2}, DDPStrategy, "ddp", CUDAAccelerator, 2), ({"strategy": "ddp", "accelerator": "cpu", "devices": 2}, DDPStrategy, "ddp", CPUAccelerator, 2), ( {"strategy": "ddp_spawn", "accelerator": "cpu", "devices": 2}, @@ -2001,7 +2001,7 @@ def training_step(self, batch, batch_idx): {"strategy": "ddp_fully_sharded", "accelerator": "gpu", "devices": 1}, DDPFullyShardedStrategy, "ddp_fully_sharded", - GPUAccelerator, + CUDAAccelerator, 1, ), ( @@ -2015,65 +2015,65 @@ def training_step(self, batch, batch_idx): {"strategy": DDPSpawnStrategy(), "accelerator": "gpu", "devices": 2}, DDPSpawnStrategy, "ddp_spawn", - GPUAccelerator, + CUDAAccelerator, 2, ), ({"strategy": DDPStrategy()}, DDPStrategy, "ddp", CPUAccelerator, 1), - ({"strategy": DDPStrategy(), "accelerator": "gpu", "devices": 2}, DDPStrategy, "ddp", GPUAccelerator, 2), + ({"strategy": DDPStrategy(), "accelerator": "gpu", "devices": 2}, DDPStrategy, "ddp", CUDAAccelerator, 2), ( {"strategy": DataParallelStrategy(), "accelerator": "gpu", "devices": 2}, DataParallelStrategy, "dp", - GPUAccelerator, + CUDAAccelerator, 2, ), ( {"strategy": DDPFullyShardedStrategy(), "accelerator": "gpu", "devices": 2}, DDPFullyShardedStrategy, "ddp_fully_sharded", - GPUAccelerator, + CUDAAccelerator, 2, ), ( {"strategy": DDPSpawnShardedStrategy(), "accelerator": "gpu", "devices": 2}, DDPSpawnShardedStrategy, "ddp_sharded_spawn", - GPUAccelerator, + CUDAAccelerator, 2, ), ( {"strategy": DDPShardedStrategy(), "accelerator": "gpu", "devices": 2}, DDPShardedStrategy, "ddp_sharded", - GPUAccelerator, + CUDAAccelerator, 2, ), ( {"strategy": "ddp_spawn", "accelerator": "gpu", "devices": 2, "num_nodes": 2}, DDPSpawnStrategy, "ddp_spawn", - GPUAccelerator, + CUDAAccelerator, 2, ), ( {"strategy": "ddp_fully_sharded", "accelerator": "gpu", "devices": 1, "num_nodes": 2}, DDPFullyShardedStrategy, "ddp_fully_sharded", - GPUAccelerator, + CUDAAccelerator, 1, ), ( {"strategy": "ddp_sharded", "accelerator": "gpu", "devices": 2, "num_nodes": 2}, DDPShardedStrategy, "ddp_sharded", - GPUAccelerator, + CUDAAccelerator, 2, ), ( {"strategy": "ddp_sharded_spawn", "accelerator": "gpu", "devices": 2, "num_nodes": 2}, DDPSpawnShardedStrategy, "ddp_sharded_spawn", - GPUAccelerator, + CUDAAccelerator, 2, ), ],