fix dataset preprocess (#2102)

modelscope · Sep 23, 2024 · 4bd62ea · 4bd62ea
1 parent a3afb9f
commit 4bd62ea
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/swift/llm/utils/dataset.py b/swift/llm/utils/dataset.py
@@ -2813,7 +2813,7 @@ def load_dataset_from_local(dataset_path_list: Optional[Union[str, List[str]]],
         dataset = preprocess_func(dataset)
         if streaming:
             dataset = dataset.to_iterable_dataset()
-        dataset_list.append(preprocess_func(dataset))
+        dataset_list.append(dataset)
 
     if len(dataset_list) == 1:
         return dataset_list[0]