Refine ORT example dataloader (#1013)

Signed-off-by: yuwenzho <[email protected]>

docs/source/metric.md

@@ -140,4 +140,4 @@ q_model = fit(model, config, calib_dataloader=calib_dataloader, eval_dataloader=
 
 - Refer to this [example](https://github.com/intel/neural-compressor/tree/master/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq_static) for how to define a customised metric.
 
-- Refer to this [example](https://github.com/intel/neural-compressor/blob/master/examples/tensorflow/image_recognition/tensorflow_models/efficientnet-b0/quantization/ptq) for how to use internal metric.
+- Refer to this [example](https://github.com/intel/neural-compressor/blob/master/examples/tensorflow/image_recognition/tensorflow_models/efficientnet-b0/quantization/ptq) for how to use internal metric.

examples/onnxrt/nlp/bert/quantization/ptq_dynamic/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

examples/onnxrt/nlp/bert/quantization/ptq_static/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = "%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -359,7 +359,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):
@@ -425,7 +425,6 @@ def eval_func(model):
             quant_format=args.quant_format,
             calibration_sampling_size=[8, 16, 32],
             recipes={"optypes_to_exclude_output_quant": ["MatMul", "Gemm", "Attention", "FusedGemm"]},
-            diagnosis=args.diagnose,
         )
         q_model = quantization.fit(model, 
                                    config,

examples/onnxrt/nlp/distilbert/quantization/ptq_dynamic/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = "%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -345,7 +345,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

examples/onnxrt/nlp/distilbert/quantization/ptq_static/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = "%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_dynamic/main.py

@@ -217,11 +217,10 @@ def eval_func(model):
         if args.mode == 'performance':
             from neural_compressor.benchmark import fit
             from neural_compressor.config import BenchmarkConfig
-            from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=4,
                                    num_of_instance=1)
-            b_dataloader = DefaultDataLoader(ds, args.eval_batch_size)
+            b_dataloader = DataLoader(ds, args.eval_batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         else:
             evaluate(args, model, tokenizer)

examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_static/main.py

@@ -216,11 +216,10 @@ def eval_func(model):
         if args.mode == 'performance':
             from neural_compressor.benchmark import fit
             from neural_compressor.config import BenchmarkConfig
-            from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=4,
                                    num_of_instance=1)
-            b_dataloader = DefaultDataLoader(ds, args.eval_batch_size)
+            b_dataloader = DataLoader(ds, args.eval_batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         else:
             evaluate(args, model, tokenizer)
@@ -251,7 +250,6 @@ def eval_func(model):
 
         from neural_compressor import quantization
         from neural_compressor.config import AccuracyCriterion, PostTrainingQuantConfig
-        from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
         from neural_compressor.utils.constant import FP32
         accuracy_criterion = AccuracyCriterion()
         accuracy_criterion.higher_is_better = False
@@ -268,7 +266,7 @@ def eval_func(model):
         q_model = quantization.fit(model, 
                                    config,
                                    eval_func=eval_func,
-                                   calib_dataloader=DefaultDataLoader(ds, args.eval_batch_size))
+                                   calib_dataloader=DataLoader(ds, args.eval_batch_size))
         q_model.save(args.output_model)
 
 

examples/onnxrt/nlp/huggingface_model/question_answering/quantization/ptq_dynamic/main.py

@@ -40,7 +40,7 @@
 from utils_model import ORTModel
 from utils_qa import postprocess_qa_predictions
 
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 
 # Will error if the minimal version of Transformers is not installed. Remove at your own risks.
@@ -493,7 +493,7 @@ def eval_func(model, *args):
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=28,
                                    num_of_instance=1)
-            b_dataloader = DefaultDataLoader(b_dataset, model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(model)

examples/onnxrt/nlp/huggingface_model/question_answering/quantization/ptq_static/main.py

@@ -39,7 +39,7 @@
 from evaluate import load
 from utils_model import ORTModel
 from utils_qa import postprocess_qa_predictions
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 
 # Will error if the minimal version of Transformers is not installed. Remove at your own risks.
@@ -499,7 +499,9 @@ def eval_func(model, *args):
         q_model = quantization.fit(model, 
                                    config,
                                    eval_func=eval_func,
-                                   calib_dataloader=DefaultDataLoader(calib_dataset, model_args.batch_size)
+                                   calib_dataloader=DataLoader(framework='onnxrt', 
+                                                               dataset=calib_dataset, 
+                                                               batch_size=model_args.batch_size)
                                    )
         q_model.save(model_args.save_path)
 
@@ -512,7 +514,7 @@ def eval_func(model, *args):
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=28,
                                    num_of_instance=1)
-            b_dataloader = DefaultDataLoader(b_dataset, model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(model)

examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_dynamic/main.py

@@ -26,8 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
-from neural_compressor.data.datasets.dummy_dataset import DummyDataset
+from neural_compressor.data import DataLoader
 
 
 class ONNXRTBertDataset:
@@ -348,7 +347,7 @@ def result(self):
                                 data_dir=args.data_path,
                                 model_name_or_path=args.model_name_or_path,
                                 task=args.task)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model, *args):

examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_static/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 
 class ONNXRTBertDataset:
@@ -354,7 +354,7 @@ def result(self):
                                 data_dir=args.data_path,
                                 model_name_or_path=args.model_name_or_path,
                                 task=args.task)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model, *args):

examples/onnxrt/nlp/huggingface_model/token_classification/layoutlmft/quantization/ptq_dynamic/main.py

@@ -28,7 +28,7 @@
 from transformers.utils import check_min_version
 import onnxruntime
 import onnx
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 
 # Will error if the minimal version of Transformers is not installed. Remove at your own risks.
@@ -456,7 +456,7 @@ def eval_func(model):
                 conf = BenchmarkConfig(iteration=100,
                                         cores_per_instance=28,
                                         num_of_instance=1)
-                b_dataloader = DefaultDataLoader(b_dataset, model_args.batch_size)
+                b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
                 fit(onnx_model, conf, b_dataloader=b_dataloader)
             elif model_args.mode == 'accuracy':
                 eval_f1 = eval_func(onnx_model)

examples/onnxrt/nlp/huggingface_model/token_classification/layoutlmft/quantization/ptq_static/main.py

@@ -28,7 +28,7 @@
 from transformers.utils import check_min_version
 import onnxruntime
 import onnx
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 
 # Will error if the minimal version of Transformers is not installed. Remove at your own risks.
@@ -457,7 +457,9 @@ def eval_func(model):
             q_model = quantization.fit(onnx_model, 
                                        config,
                                        eval_func=eval_func,
-                                       calib_dataloader=DefaultDataLoader(calib_dataset, 1))
+                                       calib_dataloader=DataLoader(framework='onnxrt',
+                                                                   dataset=calib_dataset, 
+                                                                   batch_size=1))
             q_model.save(model_args.save_path)
         if model_args.benchmark:
             onnx_model = onnx.load(model_args.input_model)
@@ -468,7 +470,7 @@ def eval_func(model):
                 conf = BenchmarkConfig(iteration=100,
                                         cores_per_instance=28,
                                         num_of_instance=1)
-                b_dataloader = DefaultDataLoader(b_dataset, model_args.batch_size)
+                b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
                 fit(onnx_model, conf, b_dataloader=b_dataloader)
             elif model_args.mode == 'accuracy':
                 eval_f1 = eval_func(onnx_model)

examples/onnxrt/nlp/huggingface_model/token_classification/layoutlmv3/quantization/ptq_dynamic/main.py

@@ -37,7 +37,7 @@
 import onnxruntime
 import onnx
 
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 
@@ -495,7 +495,7 @@ def eval_func(model):
             conf = BenchmarkConfig(iteration=100,
                                     cores_per_instance=28,
                                     num_of_instance=1)
-            b_dataloader = DefaultDataLoader(b_dataset, model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(onnx_model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(onnx_model)

examples/onnxrt/nlp/huggingface_model/token_classification/layoutlmv3/quantization/ptq_static/main.py

@@ -37,7 +37,7 @@
 import onnxruntime
 import onnx
 
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 
@@ -493,7 +493,9 @@ def eval_func(model):
         q_model = quantization.fit(onnx_model, 
                                     config,
                                     eval_func=eval_func,
-                                    calib_dataloader=DefaultDataLoader(calib_dataset, 1))
+                                    calib_dataloader=DataLoader(framework='onnxrt', 
+                                                                dataset=calib_dataset, 
+                                                                batch_size=1))
         q_model.save(model_args.save_path)
 
     if model_args.benchmark:
@@ -505,7 +507,7 @@ def eval_func(model):
             conf = BenchmarkConfig(iteration=100,
                                     cores_per_instance=28,
                                     num_of_instance=1,)
-            b_dataloader = DefaultDataLoader(b_dataset, model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(onnx_model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(onnx_model)

examples/onnxrt/nlp/mobilebert/quantization/ptq_dynamic/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = "%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

examples/onnxrt/nlp/mobilebert/quantization/ptq_static/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = "%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -359,7 +359,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

examples/onnxrt/nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/gpt2.py

@@ -239,11 +239,11 @@ def eval_func(model):
         if args.mode == 'performance':
             from neural_compressor.benchmark import fit
             from neural_compressor.config import BenchmarkConfig
-            from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+            from neural_compressor.data import DataLoader
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=4,
                                    num_of_instance=1)
-            b_dataloader = DefaultDataLoader(ds, args.eval_batch_size)
+            b_dataloader = DataLoader(framework='onnxrt', dataset=ds, batch_size=args.eval_batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         else:
             evaluate(args, model, tokenizer)

examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = "%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

examples/onnxrt/nlp/roberta/quantization/ptq_static/main.py

@@ -26,7 +26,7 @@
 import numpy as np
 from dataclasses import dataclass
 from typing import List, Optional, Union
-from neural_compressor.data.dataloaders.onnxrt_dataloader import DefaultDataLoader
+from neural_compressor.data import DataLoader
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format = "%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -359,7 +359,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DefaultDataLoader(dataset, args.batch_size)
+    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):