From 2a1a5a0807de9f863f313981f6537bd2c6202d0d Mon Sep 17 00:00:00 2001 From: michaelfeil Date: Tue, 2 Apr 2024 22:53:08 -0700 Subject: [PATCH 1/7] add weight utils --- vllm/model_executor/weight_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py index 9181f298871db..02086ca3f632d 100644 --- a/vllm/model_executor/weight_utils.py +++ b/vllm/model_executor/weight_utils.py @@ -8,6 +8,7 @@ from typing import Any, Iterator, List, Optional, Tuple import filelock +import huggingface_hub.constants import numpy as np import torch from huggingface_hub import HfFileSystem, snapshot_download @@ -28,6 +29,13 @@ temp_dir = os.environ.get('TMPDIR') or os.environ.get( 'TEMP') or os.environ.get('TMP') or "/tmp/" +try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True +except ImportError: + pass + class Disabledtqdm(tqdm): From 61d2373e98b2cbb6a6b57aeeb356cbeff8d4d3d3 Mon Sep 17 00:00:00 2001 From: michaelfeil Date: Wed, 3 Apr 2024 09:32:34 -0700 Subject: [PATCH 2/7] update hf_transfer activation --- tests/model_executor/weight_utils.py | 15 +++++++++++++++ vllm/model_executor/weight_utils.py | 18 ++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 tests/model_executor/weight_utils.py diff --git a/tests/model_executor/weight_utils.py b/tests/model_executor/weight_utils.py new file mode 100644 index 0000000000000..e333e075a5892 --- /dev/null +++ b/tests/model_executor/weight_utils.py @@ -0,0 +1,15 @@ +import huggingface_hub.constants +from vllm.model_executor.weight_utils import enable_hf_transfer + +def test_hf_transfer_auto_activation(): + enable_hf_transfer() + try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + HF_TRANFER_ACTIVE = True + except ImportError: + HF_TRANFER_ACTIVE = False + assert huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER == HF_TRANFER_ACTIVE + +if __name__ == "__main__": + test_hf_transfer_auto_activation() \ No newline at end of file diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py index 02086ca3f632d..be19bf61fbb0d 100644 --- a/vllm/model_executor/weight_utils.py +++ b/vllm/model_executor/weight_utils.py @@ -29,12 +29,18 @@ temp_dir = os.environ.get('TMPDIR') or os.environ.get( 'TEMP') or os.environ.get('TMP') or "/tmp/" -try: - # enable hf hub transfer if available - import hf_transfer # type: ignore # noqa - huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True -except ImportError: - pass + +def enable_hf_transfer(): + """enforces activation of hf_transfer if installed.""" + try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True + except ImportError: + pass + + +enable_hf_transfer() class Disabledtqdm(tqdm): From ddb3f50c2aa593ea27351233ffa8a64cd445b1e8 Mon Sep 17 00:00:00 2001 From: michaelfeil Date: Wed, 3 Apr 2024 09:43:24 -0700 Subject: [PATCH 3/7] fix length for ruff --- tests/model_executor/weight_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/model_executor/weight_utils.py b/tests/model_executor/weight_utils.py index e333e075a5892..f49991b1cc522 100644 --- a/tests/model_executor/weight_utils.py +++ b/tests/model_executor/weight_utils.py @@ -1,6 +1,8 @@ -import huggingface_hub.constants +import huggingface_hub.constants + from vllm.model_executor.weight_utils import enable_hf_transfer + def test_hf_transfer_auto_activation(): enable_hf_transfer() try: @@ -9,7 +11,9 @@ def test_hf_transfer_auto_activation(): HF_TRANFER_ACTIVE = True except ImportError: HF_TRANFER_ACTIVE = False - assert huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER == HF_TRANFER_ACTIVE + assert (huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER == + HF_TRANFER_ACTIVE) + if __name__ == "__main__": - test_hf_transfer_auto_activation() \ No newline at end of file + test_hf_transfer_auto_activation() From 2ef154e87be2e84e09e9945cae3c780a14714f96 Mon Sep 17 00:00:00 2001 From: michaelfeil Date: Wed, 3 Apr 2024 18:45:45 -0700 Subject: [PATCH 4/7] add weight utils with env flag --- tests/model_executor/weight_utils.py | 7 +++++++ vllm/model_executor/weight_utils.py | 19 ++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/model_executor/weight_utils.py b/tests/model_executor/weight_utils.py index f49991b1cc522..3154f2826d10c 100644 --- a/tests/model_executor/weight_utils.py +++ b/tests/model_executor/weight_utils.py @@ -1,9 +1,16 @@ +import os + import huggingface_hub.constants +import pytest from vllm.model_executor.weight_utils import enable_hf_transfer def test_hf_transfer_auto_activation(): + if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ: + # in case it is already set, we can't test the auto activation + pytest.skip( + "HF_HUB_ENABLE_HF_TRANSFER is set, can't test auto activation") enable_hf_transfer() try: # enable hf hub transfer if available diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py index be19bf61fbb0d..c59a6ce5b8dc2 100644 --- a/vllm/model_executor/weight_utils.py +++ b/vllm/model_executor/weight_utils.py @@ -31,13 +31,18 @@ def enable_hf_transfer(): - """enforces activation of hf_transfer if installed.""" - try: - # enable hf hub transfer if available - import hf_transfer # type: ignore # noqa - huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True - except ImportError: - pass + """automatically activates hf_transfer + """ + if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ: + # no additional logic, use default behaviour. + return + else: + try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True + except ImportError: + pass enable_hf_transfer() From d240d0900adca15b4a37f727bcd18391492446f4 Mon Sep 17 00:00:00 2001 From: Michael Feil <63565275+michaelfeil@users.noreply.github.com> Date: Wed, 3 Apr 2024 19:20:15 -0700 Subject: [PATCH 5/7] Update vllm/model_executor/weight_utils.py Co-authored-by: Nick Hill --- vllm/model_executor/weight_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py index c59a6ce5b8dc2..a9ce95163999d 100644 --- a/vllm/model_executor/weight_utils.py +++ b/vllm/model_executor/weight_utils.py @@ -33,10 +33,7 @@ def enable_hf_transfer(): """automatically activates hf_transfer """ - if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ: - # no additional logic, use default behaviour. - return - else: + if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ: try: # enable hf hub transfer if available import hf_transfer # type: ignore # noqa From 5950a80e3c13ade98cb804857eefdc5258abd958 Mon Sep 17 00:00:00 2001 From: michaelfeil Date: Wed, 3 Apr 2024 19:29:15 -0700 Subject: [PATCH 6/7] run ci --- vllm/model_executor/weight_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py index a9ce95163999d..bd2bb71182e66 100644 --- a/vllm/model_executor/weight_utils.py +++ b/vllm/model_executor/weight_utils.py @@ -35,7 +35,7 @@ def enable_hf_transfer(): """ if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ: try: - # enable hf hub transfer if available + # enable hf hub transfer if available import hf_transfer # type: ignore # noqa huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True except ImportError: From c20ec0b6db8d8a728048a5403e69ffbbf53318a3 Mon Sep 17 00:00:00 2001 From: michaelfeil Date: Wed, 3 Apr 2024 19:29:32 -0700 Subject: [PATCH 7/7] run ci v2 --- vllm/model_executor/weight_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py index bd2bb71182e66..a9ce95163999d 100644 --- a/vllm/model_executor/weight_utils.py +++ b/vllm/model_executor/weight_utils.py @@ -35,7 +35,7 @@ def enable_hf_transfer(): """ if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ: try: - # enable hf hub transfer if available + # enable hf hub transfer if available import hf_transfer # type: ignore # noqa huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True except ImportError: