Refactor text data loading (#870)

Lightning-Universe · Oct 18, 2021 · a94ed6c · a94ed6c
1 parent 4f608b0
commit a94ed6c
Show file tree

Hide file tree

Showing 7 changed files with 679 additions and 250 deletions.
diff --git a/docs/source/api/text.rst b/docs/source/api/text.rst
@@ -22,12 +22,14 @@ ______________
 
     classification.data.TextClassificationPostprocess
     classification.data.TextClassificationPreprocess
-    classification.data.TextCSVDataSource
-    classification.data.TextDataSource
     classification.data.TextDeserializer
-    classification.data.TextFileDataSource
+    classification.data.TextDataSource
+    classification.data.TextCSVDataSource
     classification.data.TextJSONDataSource
-    classification.data.TextSentencesDataSource
+    classification.data.TextDataFrameDataSource
+    classification.data.TextParquetDataSource
+    classification.data.TextHuggingFaceDatasetDataSource
+    classification.data.TextListDataSource
 
 Question Answering
 __________________

diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py
@@ -158,11 +158,12 @@ class DefaultDataSources(LightningEnum):
     TENSORS = "tensors"
     CSV = "csv"
     JSON = "json"
+    PARQUET = "parquet"
     DATASETS = "datasets"
+    HUGGINGFACE_DATASET = "hf_dataset"
     FIFTYONE = "fiftyone"
     DATAFRAME = "data_frame"
     LISTS = "lists"
-    SENTENCES = "sentences"
     LABELSTUDIO = "labelstudio"
 
     # TODO: Create a FlashEnum class???