From ca07d0af8524927752b0499a7861d4d932ea9135 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 27 Jun 2024 14:12:56 +0800 Subject: [PATCH 1/2] fix --- .../DPO\350\256\255\347\273\203\346\226\207\346\241\243.md" | 2 +- docs/source_en/LLM/DPO.md | 2 +- swift/llm/utils/utils.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git "a/docs/source/LLM/DPO\350\256\255\347\273\203\346\226\207\346\241\243.md" "b/docs/source/LLM/DPO\350\256\255\347\273\203\346\226\207\346\241\243.md" index fa5a95361..172fe650d 100644 --- "a/docs/source/LLM/DPO\350\256\255\347\273\203\346\226\207\346\241\243.md" +++ "b/docs/source/LLM/DPO\350\256\255\347\273\203\346\226\207\346\241\243.md" @@ -80,7 +80,7 @@ cd examples/pytorch/llm - 如果用带有history的数据训练base模型，需要指定支持多轮对话的template(base模型往往不支持多轮对话)，对于这种情况我们默认设置了`chatml`template，你也可以支持--model_type 来选择训练模型的template - 我们默认在训练时设置`--gradient_checkpointing true`来**节约显存**, 这会略微降低训练速度. - 如果你使用的是**V100**等较老的GPU, 你需要设置`--dtype AUTO`或者`--dtype fp16`, 因为其不支持bf16. -- 如果你的机器是A100等高性能显卡, 且使用的是qwen系列模型, 推荐你安装[**flash-attn**](https://github.com/Dao-AILab/flash-attention), 这将会加快训练和推理的速度以及显存占用(A10, 3090, V100等显卡不支持flash-attn进行训练). 支持flash-attn的模型可以查看[LLM支持的模型](支持的模型和数据集.md#模型) +- 如果你的机器是A100等高性能显卡, 且使用的是qwen系列模型, 推荐你安装[**flash-attn**](https://github.com/Dao-AILab/flash-attention), 这将会加快训练和推理的速度以及显存占用(3090, V100等显卡不支持flash-attn进行训练). 支持flash-attn的模型可以查看[LLM支持的模型](支持的模型和数据集.md#模型) - 如果你需要断网进行训练, 请使用`--model_id_or_path `和设置`--check_model_is_latest false`. 具体参数含义请查看[命令行参数](命令行参数.md). - 如果你想在训练时, 将权重push到ModelScope Hub中, 你需要设置`--push_to_hub true`. diff --git a/docs/source_en/LLM/DPO.md b/docs/source_en/LLM/DPO.md index 0bae5a075..ee8c013a5 100644 --- a/docs/source_en/LLM/DPO.md +++ b/docs/source_en/LLM/DPO.md @@ -78,7 +78,7 @@ cd examples/pytorch/llm - We default to setting `--gradient_checkpointing true` during training to **save memory**, which will slightly reduce training speed. - If you are using older GPUs such as **V100**, you need to set `--dtype AUTO` or `--dtype fp16`, because they do not support bf16. -- If your machine has high-performance graphics cards like A100 and you are using the qwen series models, we recommend installing [**flash-attn**](https://github.com/Dao-AILab/flash-attention), which will speed up training and inference as well as reduce memory usage (A10, 3090, V100, etc. graphics cards do not support training with flash-attn). Models that support flash-attn can be viewed in [LLM Supported Models](Supported-models-datasets.md#models) +- If your machine has high-performance graphics cards like A100 and you are using the qwen series models, we recommend installing [**flash-attn**](https://github.com/Dao-AILab/flash-attention), which will speed up training and inference as well as reduce memory usage (3090, V100, etc. graphics cards do not support training with flash-attn). Models that support flash-attn can be viewed in [LLM Supported Models](Supported-models-datasets.md#models) - If you need to train offline, please use `--model_id_or_path ` and set `--check_model_is_latest false`. For specific parameter meanings, please see [Command Line Arguments](Command-line-parameters.md). - If you want to push weights to the ModelScope Hub during training, you need to set `--push_to_hub true`. diff --git a/swift/llm/utils/utils.py b/swift/llm/utils/utils.py index b5a44c857..103c98198 100644 --- a/swift/llm/utils/utils.py +++ b/swift/llm/utils/utils.py @@ -269,7 +269,11 @@ def _try_fetch(self, first_idx: int) -> Optional[Dict[str, Any]]: idx = np.random.permutation(len(self))[:self.try_fetch_time - 1] for i in [first_idx] + idx.tolist(): data = self.dataset[i] - res = self.template.encode(data) + from PIL import UnidentifiedImageError + try: + res = self.template.encode(data) + except OSError: + continue if len(res[0]) > 0: return res From 26c77e60330837f443aede5811b421ab36f28ccf Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 27 Jun 2024 14:19:24 +0800 Subject: [PATCH 2/2] remove useless code --- swift/llm/utils/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/swift/llm/utils/utils.py b/swift/llm/utils/utils.py index 103c98198..ab28d3c55 100644 --- a/swift/llm/utils/utils.py +++ b/swift/llm/utils/utils.py @@ -269,7 +269,6 @@ def _try_fetch(self, first_idx: int) -> Optional[Dict[str, Any]]: idx = np.random.permutation(len(self))[:self.try_fetch_time - 1] for i in [first_idx] + idx.tolist(): data = self.dataset[i] - from PIL import UnidentifiedImageError try: res = self.template.encode(data) except OSError: