From ae6b3bedddb072317994eebc94954e16e5419bcf Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Mon, 29 Apr 2024 13:14:15 -0400 Subject: [PATCH 01/11] check model.device --- src/transformers/pipelines/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 4bb5cffb1287a9..e9a9b992082363 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -841,6 +841,8 @@ def __init__( if hf_device_map is not None: # Take the first device used by `accelerate`. device = next(iter(hf_device_map.values())) + elif self.model.device is not None: + device = self.model.device else: device = -1 From b3113baab4db2dce99a7f5fbec34abdb96b35daf Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Mon, 29 Apr 2024 13:30:57 -0400 Subject: [PATCH 02/11] fix --- src/transformers/pipelines/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index e9a9b992082363..b6f8289b73431a 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -830,7 +830,8 @@ def __init__( # `accelerate` device map hf_device_map = getattr(self.model, "hf_device_map", None) - + model_device = getattr(self.model, "device", None) + if hf_device_map is not None and device is not None: raise ValueError( "The model has been loaded with `accelerate` and therefore cannot be moved to a specific device. Please " @@ -841,8 +842,8 @@ def __init__( if hf_device_map is not None: # Take the first device used by `accelerate`. device = next(iter(hf_device_map.values())) - elif self.model.device is not None: - device = self.model.device + elif model_device is not None: + device = model_device else: device = -1 From 03a87fb0f124feb924983ff067298c4deb43083e Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Mon, 29 Apr 2024 13:31:54 -0400 Subject: [PATCH 03/11] style fix --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index b6f8289b73431a..9b2989e34c15ce 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -831,7 +831,7 @@ def __init__( # `accelerate` device map hf_device_map = getattr(self.model, "hf_device_map", None) model_device = getattr(self.model, "device", None) - + if hf_device_map is not None and device is not None: raise ValueError( "The model has been loaded with `accelerate` and therefore cannot be moved to a specific device. Please " From be79f6661003216fe8e399e37b118e2d2b283708 Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Tue, 30 Apr 2024 15:32:09 -0400 Subject: [PATCH 04/11] move model device --- src/transformers/pipelines/base.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 9b2989e34c15ce..327adac3ee1138 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -830,7 +830,6 @@ def __init__( # `accelerate` device map hf_device_map = getattr(self.model, "hf_device_map", None) - model_device = getattr(self.model, "device", None) if hf_device_map is not None and device is not None: raise ValueError( @@ -842,12 +841,12 @@ def __init__( if hf_device_map is not None: # Take the first device used by `accelerate`. device = next(iter(hf_device_map.values())) - elif model_device is not None: - device = model_device else: device = -1 if is_torch_available() and self.framework == "pt": + if device is None and self.model.device is not None: + device = self.model.device if isinstance(device, torch.device): if device.type == "xpu" and not is_torch_xpu_available(check_device=True): raise ValueError(f'{device} is not available, you should use device="cpu" instead') @@ -874,14 +873,14 @@ def __init__( self.device = device if device is not None else -1 self.binary_output = binary_output - # We shouldn't call `model.to()` for models loaded with accelerate if ( self.framework == "pt" - and self.device is not None and not (isinstance(self.device, int) and self.device < 0) and hf_device_map is None + and self.model.device != self.device ): + print("______MOVE_____") self.model.to(self.device) # Update config and generation_config with task specific parameters From dea4dbb4e73cbaf23ff28573253366814303c4d5 Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Tue, 30 Apr 2024 15:33:23 -0400 Subject: [PATCH 05/11] remove print --- src/transformers/pipelines/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 327adac3ee1138..414833377a3fb0 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -876,11 +876,10 @@ def __init__( # We shouldn't call `model.to()` for models loaded with accelerate if ( self.framework == "pt" + and self.model.device != self.device and not (isinstance(self.device, int) and self.device < 0) and hf_device_map is None - and self.model.device != self.device ): - print("______MOVE_____") self.model.to(self.device) # Update config and generation_config with task specific parameters From 385e8bd217b0e3925c253b047b7a48f703421fe7 Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Tue, 30 Apr 2024 15:41:09 -0400 Subject: [PATCH 06/11] add comment --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 414833377a3fb0..a85622aa96343a 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -873,7 +873,7 @@ def __init__( self.device = device if device is not None else -1 self.binary_output = binary_output - # We shouldn't call `model.to()` for models loaded with accelerate + # We shouldn't call `model.to()` for models loaded with accelerate as well as the case that model is already on device if ( self.framework == "pt" and self.model.device != self.device From c1f9440d6c3dd02f7cb1e2de3dc394cf927ceb0b Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Tue, 30 Apr 2024 15:49:23 -0400 Subject: [PATCH 07/11] fix --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index a85622aa96343a..b318e1b12b414c 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -845,7 +845,7 @@ def __init__( device = -1 if is_torch_available() and self.framework == "pt": - if device is None and self.model.device is not None: + if device == -1 and self.model.device is not None: device = self.model.device if isinstance(device, torch.device): if device.type == "xpu" and not is_torch_xpu_available(check_device=True): From 83474728851ccbce14c21848368f9bd86f0c631b Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Sat, 11 May 2024 08:29:26 -0400 Subject: [PATCH 08/11] add unit test --- tests/pipelines/test_pipelines_common.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index c680b4c634de40..77a0c3cd905e85 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -519,6 +519,24 @@ def test_pipeline_negative_device(self): actual_output = classifier("Test input.") self.assertEqual(expected_output, actual_output) + def test_pipeline_device_equal_model_device(self): + import torch + + from transformers import AutoModelForCausalLM + + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") + # test no device passed to pipeline + model = AutoModelForCausalLM.from_pretrained( + "hf-internal-testing/tiny-random-bert", torch_dtype=torch.float16 + ).to(torch_device) + model_device = model.device + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + self.assertEqual(model_device, pipe.model.device) + # test when device ids are different, pipeline should follow the passed device + model = model.to(f"{torch_device}:1") + pipe = pipeline("text-generation", model=model, device=f"{torch_device}:0", tokenizer=tokenizer) + assert pipe.device == torch.device(f"{torch_device}:0") + @slow @require_torch def test_load_default_pipelines_pt(self): From 868c1eef86cc814c3630d416a384bf753a4b5210 Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Sat, 11 May 2024 09:02:01 -0400 Subject: [PATCH 09/11] optimize --- tests/pipelines/test_pipelines_common.py | 30 ++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 77a0c3cd905e85..65d1a28ebca623 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -48,6 +48,7 @@ require_tf, require_torch, require_torch_accelerator, + require_torch_multi_accelerator, require_torch_or_tf, slow, torch_device, @@ -519,23 +520,38 @@ def test_pipeline_negative_device(self): actual_output = classifier("Test input.") self.assertEqual(expected_output, actual_output) - def test_pipeline_device_equal_model_device(self): + @require_torch_accelerator + def test_pipeline_should_not_move_model(self): + # Test when model is already on device, pipeline shouldn't move model import torch from transformers import AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") - # test no device passed to pipeline model = AutoModelForCausalLM.from_pretrained( "hf-internal-testing/tiny-random-bert", torch_dtype=torch.float16 ).to(torch_device) model_device = model.device + # No device passed to pipeline pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) - self.assertEqual(model_device, pipe.model.device) - # test when device ids are different, pipeline should follow the passed device - model = model.to(f"{torch_device}:1") - pipe = pipeline("text-generation", model=model, device=f"{torch_device}:0", tokenizer=tokenizer) - assert pipe.device == torch.device(f"{torch_device}:0") + self.assertEqual(pipe.model.device, model_device) + + @require_torch_multi_accelerator + def test_pipeline_should_move_model(self): + # Test when device ids are different, pipeline should move the model to the passed device id + import torch + + from transformers import AutoModelForCausalLM + + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") + model_device = f"{torch_device}:1" + model = AutoModelForCausalLM.from_pretrained( + "hf-internal-testing/tiny-random-bert", torch_dtype=torch.float16 + ).to(model_device) + target_device = f"{torch_device}:0" + assert model_device != model_device + pipe = pipeline("text-generation", model=model, device=target_device, tokenizer=tokenizer) + assert pipe.model.device == torch.device(target_device) @slow @require_torch From f80667df8c5928e04d25566c0e6e95483953e408 Mon Sep 17 00:00:00 2001 From: "Lin, Fanli" Date: Sat, 11 May 2024 09:30:53 -0400 Subject: [PATCH 10/11] change test names and add more cases --- tests/pipelines/test_pipelines_common.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 65d1a28ebca623..6bcc74215672b9 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -521,23 +521,36 @@ def test_pipeline_negative_device(self): self.assertEqual(expected_output, actual_output) @require_torch_accelerator - def test_pipeline_should_not_move_model(self): - # Test when model is already on device, pipeline shouldn't move model + def test_pipeline_no_device(self): + # Test when no device is passed to pipeline import torch from transformers import AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") + # Case 1: Model is manually moved to device model = AutoModelForCausalLM.from_pretrained( "hf-internal-testing/tiny-random-bert", torch_dtype=torch.float16 ).to(torch_device) model_device = model.device - # No device passed to pipeline pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) self.assertEqual(pipe.model.device, model_device) + # Case 2: Model is loaded by accelerate + model = AutoModelForCausalLM.from_pretrained( + "hf-internal-testing/tiny-random-bert", device_map=torch_device, torch_dtype=torch.float16 + ) + model_device = model.device + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + self.assertEqual(pipe.model.device, model_device) + # Case 3: device_map is passed to model and device is passed to pipeline + model = AutoModelForCausalLM.from_pretrained( + "hf-internal-testing/tiny-random-bert", device_map=torch_device, torch_dtype=torch.float16 + ) + with self.assertRaises(ValueError): + pipe = pipeline("text-generation", model=model, device="cpu", tokenizer=tokenizer) @require_torch_multi_accelerator - def test_pipeline_should_move_model(self): + def test_pipeline_device_not_equal_model_device(self): # Test when device ids are different, pipeline should move the model to the passed device id import torch @@ -549,7 +562,7 @@ def test_pipeline_should_move_model(self): "hf-internal-testing/tiny-random-bert", torch_dtype=torch.float16 ).to(model_device) target_device = f"{torch_device}:0" - assert model_device != model_device + assert model_device != target_device pipe = pipeline("text-generation", model=model, device=target_device, tokenizer=tokenizer) assert pipe.model.device == torch.device(target_device) From ff340cb14a9778fa1fd8766ef915f2d9c7d653e3 Mon Sep 17 00:00:00 2001 From: Fanli Lin Date: Mon, 13 May 2024 21:31:09 +0800 Subject: [PATCH 11/11] Update tests/pipelines/test_pipelines_common.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- tests/pipelines/test_pipelines_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 6bcc74215672b9..763c7d1a883314 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -562,9 +562,9 @@ def test_pipeline_device_not_equal_model_device(self): "hf-internal-testing/tiny-random-bert", torch_dtype=torch.float16 ).to(model_device) target_device = f"{torch_device}:0" - assert model_device != target_device + self.assertNotEqual(model_device, target_device) pipe = pipeline("text-generation", model=model, device=target_device, tokenizer=tokenizer) - assert pipe.model.device == torch.device(target_device) + self.assertEqual(pipe.model.device, torch.device(target_device)) @slow @require_torch