diff --git a/examples/tensorflow/graph_networks/graphsage/quantization/ptq/main.py b/examples/tensorflow/graph_networks/graphsage/quantization/ptq/main.py index 9ec710fe9f8..51a9e0fd384 100644 --- a/examples/tensorflow/graph_networks/graphsage/quantization/ptq/main.py +++ b/examples/tensorflow/graph_networks/graphsage/quantization/ptq/main.py @@ -138,7 +138,7 @@ def eval_func(size, output_tensor, minibatch, test): labels.append(batch_labels) iter_num += 1 total_time += time_consume - if iteration and iter_num >= iteration: + if iteration != -1 and iter_num >= iteration: break tf_logging.warn('\n---> Stop iteration {0}'.format(str(iter_num))) val_preds = np.vstack(val_preds) diff --git a/examples/tensorflow/image_recognition/tensorflow_models/vision_transformer/quantization/ptq/main.py b/examples/tensorflow/image_recognition/tensorflow_models/vision_transformer/quantization/ptq/main.py index fb6ba483795..d0010c948a1 100644 --- a/examples/tensorflow/image_recognition/tensorflow_models/vision_transformer/quantization/ptq/main.py +++ b/examples/tensorflow/image_recognition/tensorflow_models/vision_transformer/quantization/ptq/main.py @@ -52,7 +52,7 @@ arg_parser.add_argument('--int8', dest='int8', action='store_true', help='whether to use int8 model for benchmark') args = arg_parser.parse_args() -def evaluate(model, eval_dataloader, metric, postprocess=None): +def evaluate(model, eval_dataloader, postprocess=None): """Custom evaluate function to estimate the accuracy of the model. Args: @@ -61,12 +61,14 @@ def evaluate(model, eval_dataloader, metric, postprocess=None): Returns: accuracy (float): evaluation result, the larger is better. """ + from neural_compressor import METRICS from neural_compressor.model import Model model = Model(model) input_tensor = model.input_tensor output_tensor = model.output_tensor if len(model.output_tensor)>1 else \ model.output_tensor[0] iteration = -1 + metric = METRICS('tensorflow')['topk']() if args.benchmark and args.mode == 'performance': iteration = args.iters @@ -136,9 +138,6 @@ def run(self): accuracy_criterion = AccuracyCriterion(tolerable_loss=0.01), op_type_dict={'conv2d':{ 'weight':{'dtype':['fp32']}, 'activation':{'dtype':['fp32']} }} ) - from neural_compressor import METRICS - metrics = METRICS('tensorflow') - top1 = metrics['topk']() from tensorflow.core.protobuf import saved_model_pb2 sm = saved_model_pb2.SavedModel() with tf.io.gfile.GFile(args.input_graph, "rb") as f: @@ -147,10 +146,9 @@ def run(self): from neural_compressor.data import TensorflowShiftRescale postprocess = TensorflowShiftRescale() def eval(model): - return evaluate(model, eval_dataloader, top1, postprocess) - q_model = quantization.fit(graph_def, conf=conf, calib_dataloader=calib_dataloader, - # eval_dataloader=eval_dataloader, eval_metric=top1) - eval_func=eval) + return evaluate(model, eval_dataloader, postprocess) + q_model = quantization.fit(graph_def, conf=conf, eval_func=eval, + calib_dataloader=calib_dataloader) q_model.save(args.output_graph) if args.benchmark: @@ -163,9 +161,6 @@ def eval(model): 'filter': None } dataloader = create_dataloader('tensorflow', dataloader_args) - from neural_compressor import METRICS - metrics = METRICS('tensorflow') - top1 = metrics['topk']() if args.int8 or args.input_graph.endswith("-tune.pb"): input_graph = args.input_graph @@ -180,7 +175,7 @@ def eval(model): from neural_compressor.data import TensorflowShiftRescale postprocess = TensorflowShiftRescale() def eval(model): - return evaluate(model, dataloader, top1, postprocess) + return evaluate(model, dataloader, postprocess) if args.mode == 'performance': from neural_compressor.benchmark import fit diff --git a/examples/tensorflow/nlp/bert_large_squad/quantization/ptq/tune_squad.py b/examples/tensorflow/nlp/bert_large_squad/quantization/ptq/tune_squad.py index e9cd724de79..b999538ddf8 100644 --- a/examples/tensorflow/nlp/bert_large_squad/quantization/ptq/tune_squad.py +++ b/examples/tensorflow/nlp/bert_large_squad/quantization/ptq/tune_squad.py @@ -51,7 +51,7 @@ flags.DEFINE_integer("iters", 100, "The iteration used for benchmark.") -def evaluate(model, dataloader, metric, postprocess): +def evaluate(model, dataloader, postprocess): """Custom evaluate function to estimate the accuracy of the bert model. Args: @@ -60,6 +60,7 @@ def evaluate(model, dataloader, metric, postprocess): Returns: accuracy (float): evaluation result, the larger is better. """ + from neural_compressor.metric import SquadF1 from neural_compressor.adaptor.tf_utils.util import iterator_sess_run from neural_compressor.objective import Performance from neural_compressor.model import Model, BaseModel @@ -70,12 +71,12 @@ def evaluate(model, dataloader, metric, postprocess): input_tensor = model.input_tensor output_tensor = model.output_tensor if len(model.output_tensor)>1 else \ model.output_tensor[0] + warmup = 5 iteration = -1 + metric = SquadF1() + measurer = Performance() if FLAGS.benchmark and FLAGS.mode == 'performance': iteration = FLAGS.iters - measurer = Performance() - - warmup = 5 for idx, (inputs, labels) in enumerate(dataloader): # dataloader should keep the order and len of inputs same with input_tensor assert len(input_tensor) == len(inputs), \ @@ -125,8 +126,6 @@ def strip_iterator(graph_def): def main(_): tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) - from neural_compressor.metric import SquadF1 - metric = SquadF1() from neural_compressor.utils.create_obj_from_config import create_dataloader data_path = os.path.join(FLAGS.dataset_location, 'eval.tf_record') label_path = os.path.join(FLAGS.dataset_location, 'dev-v1.1.json') @@ -142,7 +141,7 @@ def main(_): from neural_compressor.data import TFSquadV1PostTransform postprocess = TFSquadV1PostTransform(label_file=label_path, vocab_file=vocab_path) def eval(model): - return evaluate(model, dataloader, metric, postprocess) + return evaluate(model, dataloader, postprocess) if FLAGS.benchmark: if FLAGS.mode == 'performance': from neural_compressor.benchmark import fit