Skip to content

Commit

Permalink
Update targets metrics for e2e (openvinotoolkit#2633) (openvinotoolki…
Browse files Browse the repository at this point in the history 
…t#2658)

Update metrics for e2e

- PT, Update metrics, ov models start collect parameters for FQ in
float32, was float16
- TF, increase threshold for unstable measured metrics on cuda.

138550, 138552
  • Loading branch information
@AlexanderDokuchaev
AlexanderDokuchaev authored Apr 26, 2024
1 parent e8f2386 commit 489cc09
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 44 deletions.
4 changes: 3 additions & 1 deletion tests/tensorflow/sota_checkpoints_eval.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,9 @@
"compression_description": "INT8 (per-tensor symmetric for weights, per-tensor asymmetric half-range for activations) + filter pruning 40%",
"batch_per_gpu": 15,
"target_tf": 32.61,
"target_ov": 32.47
"target_ov": 32.47,
"diff_target_tf_min": -0.15,
"diff_target_tf_max": 0.15
},
"yolo_v4_coco": {
"config": "examples/tensorflow/object_detection/configs/yolo_v4_coco.json",
Expand Down
86 changes: 43 additions & 43 deletions tests/torch/sota_checkpoints_eval.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@
"imagenet": {
"resnet50_imagenet": {
"config": "examples/torch/classification/configs/quantization/resnet50_imagenet.json",
"target_ov": 76.16,
"target_ov": 76.15,
"target_pt": 76.15,
"metric_type": "Acc@1",
"model_description": "ResNet-50"
},
"resnet50_imagenet_int8": {
"config": "examples/torch/classification/configs/quantization/resnet50_imagenet_int8.json",
"reference": "resnet50_imagenet",
"target_ov": 76.39,
"target_pt": 76.41,
"target_ov": 76.42,
"target_pt": 76.45,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_int8.pth",
"model_description": "ResNet-50",
Expand All @@ -23,7 +23,7 @@
"resnet50_imagenet_int8_per_tensor": {
"config": "examples/torch/classification/configs/quantization/resnet50_imagenet_int8_per_tensor.json",
"reference": "resnet50_imagenet",
"target_ov": 76.35,
"target_ov": 76.38,
"target_pt": 76.36,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_int8_per_tensor.pth",
Expand All @@ -37,8 +37,8 @@
"resnet50_imagenet_int4_int8": {
"config": "examples/torch/classification/configs/mixed_precision/resnet50_imagenet_mixed_int_hawq.json",
"reference": "resnet50_imagenet",
"target_ov": 75.61,
"target_pt": 75.94,
"target_ov": 75.69,
"target_pt": 75.86,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_int4_int8.pth",
"model_description": "ResNet-50",
Expand All @@ -49,8 +49,8 @@
"resnet50_imagenet_rb_sparsity_int8": {
"config": "examples/torch/classification/configs/sparsity_quantization/resnet50_imagenet_rb_sparsity_int8.json",
"reference": "resnet50_imagenet",
"target_ov": 75.39,
"target_pt": 75.41,
"target_ov": 75.36,
"target_pt": 75.42,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_rb_sparsity_int8.pth",
"model_description": "ResNet-50",
Expand All @@ -61,8 +61,8 @@
"resnet50_imagenet_rb_sparsity50_int8": {
"config": "examples/torch/classification/configs/sparsity_quantization/resnet50_imagenet_rb_sparsity50_int8.json",
"reference": "resnet50_imagenet",
"target_ov": 75.44,
"target_pt": 75.5,
"target_ov": 75.48,
"target_pt": 75.47,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_rb_sparsity50_int8.pth",
"model_description": "ResNet-50",
Expand Down Expand Up @@ -91,8 +91,8 @@
"inception_v3_imagenet_int8": {
"config": "examples/torch/classification/configs/quantization/inception_v3_imagenet_int8.json",
"reference": "inception_v3_imagenet",
"target_ov": 77.49,
"target_pt": 77.46,
"target_ov": 77.5,
"target_pt": 77.43,
"metric_type": "Acc@1",
"resume": "inception_v3_imagenet_int8.pth",
"model_description": "Inception V3",
Expand All @@ -104,8 +104,8 @@
"inception_v3_imagenet_rb_sparsity_int8": {
"config": "examples/torch/classification/configs/sparsity_quantization/inception_v3_imagenet_rb_sparsity_int8.json",
"reference": "inception_v3_imagenet",
"target_ov": 76.34,
"target_pt": 76.34,
"target_ov": 76.4,
"target_pt": 76.32,
"metric_type": "Acc@1",
"resume": "inception_v3_imagenet_rb_sparsity_int8.pth",
"model_description": "Inception V3",
Expand All @@ -124,8 +124,8 @@
"mobilenet_v2_imagenet_int8": {
"config": "examples/torch/classification/configs/quantization/mobilenet_v2_imagenet_int8.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 71.01,
"target_pt": 71.22,
"target_ov": 71.07,
"target_pt": 71.3,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_int8.pth",
"model_description": "MobileNet V2",
Expand All @@ -138,8 +138,8 @@
"mobilenet_v2_imagenet_int8_per_tensor": {
"config": "examples/torch/classification/configs/quantization/mobilenet_v2_imagenet_int8_per_tensor.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 71.17,
"target_pt": 71.26,
"target_ov": 71.23,
"target_pt": 71.28,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_int8_per_tensor.pth",
"model_description": "MobileNet V2",
Expand All @@ -152,8 +152,8 @@
"mobilenet_v2_imagenet_int4_int8": {
"config": "examples/torch/classification/configs/mixed_precision/mobilenet_v2_imagenet_mixed_int_hawq.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 70.52,
"target_pt": 70.68,
"target_ov": 70.72,
"target_pt": 70.57,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_int4_int8.pth",
"model_description": "MobileNet V2",
Expand All @@ -166,8 +166,8 @@
"mobilenet_v2_imagenet_rb_sparsity_int8": {
"config": "examples/torch/classification/configs/sparsity_quantization/mobilenet_v2_imagenet_rb_sparsity_int8.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 71.07,
"target_pt": 71.06,
"target_ov": 71.11,
"target_pt": 71.02,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_rb_sparsity_int8.pth",
"model_description": "MobileNet V2",
Expand All @@ -187,7 +187,7 @@
"mobilenet_v3_small_imagenet_int8": {
"config": "examples/torch/classification/configs/quantization/mobilenet_v3_small_imagenet_int8.json",
"reference": "mobilenet_v3_small_imagenet",
"target_ov": 66.92,
"target_ov": 66.9,
"target_pt": 66.87,
"metric_type": "Acc@1",
"resume": "mobilenet_v3_small_imagenet_int8.pth",
Expand All @@ -208,7 +208,7 @@
"squeezenet1_1_imagenet_int8": {
"config": "examples/torch/classification/configs/quantization/squeezenet1_1_imagenet_int8.json",
"reference": "squeezenet1_1_imagenet",
"target_ov": 58.15,
"target_ov": 58.2,
"target_pt": 58.28,
"metric_type": "Acc@1",
"resume": "squeezenet1_1_imagenet_int8.pth",
Expand All @@ -222,7 +222,7 @@
"squeezenet1_1_imagenet_int8_per_tensor": {
"config": "examples/torch/classification/configs/quantization/squeezenet1_1_imagenet_int8_per_tensor.json",
"reference": "squeezenet1_1_imagenet",
"target_ov": 58.06,
"target_ov": 58.07,
"target_pt": 58.14,
"metric_type": "Acc@1",
"resume": "squeezenet1_1_imagenet_int8_per_tensor.pth",
Expand All @@ -236,8 +236,8 @@
"squeezenet1_1_imagenet_int4_int8": {
"config": "examples/torch/classification/configs/mixed_precision/squeezenet1_1_imagenet_mixed_int_hawq_old_eval.json",
"reference": "squeezenet1_1_imagenet",
"target_ov": 57.53,
"target_pt": 57.66,
"target_ov": 57.63,
"target_pt": 57.63,
"metric_type": "Acc@1",
"resume": "squeezenet1_1_imagenet_int4_int8.pth",
"model_description": "SqueezeNet V1.1",
Expand All @@ -249,7 +249,7 @@
},
"resnet18_imagenet": {
"config": "examples/torch/classification/configs/pruning/resnet18_imagenet.json",
"target_ov": 69.77,
"target_ov": 69.76,
"target_pt": 69.76,
"metric_type": "Acc@1",
"model_description": "ResNet-18"
Expand Down Expand Up @@ -324,8 +324,8 @@
"ssd300_mobilenet_voc_magnitude_sparsity_int8": {
"config": "examples/torch/object_detection/configs/ssd300_mobilenet_voc_magnitude_int8.json",
"reference": "ssd300_mobilenet_voc",
"target_ov": 63.01,
"target_pt": 62.99,
"target_ov": 63.02,
"target_pt": 62.97,
"metric_type": "Mean AP",
"resume": "ssd300_mobilenet_voc_magnitude_sparsity_int8.pth",
"model_description": "SSD300-MobileNet",
Expand Down Expand Up @@ -359,8 +359,8 @@
"ssd300_vgg_voc_magnitude_sparsity_int8": {
"config": "examples/torch/object_detection/configs/ssd300_vgg_voc_magnitude_sparsity_int8.json",
"reference": "ssd300_vgg_voc",
"target_ov": 77.46,
"target_pt": 77.66,
"target_ov": 77.44,
"target_pt": 77.67,
"metric_type": "Mean AP",
"resume": "ssd300_vgg_voc_magnitude_sparsity_int8.pth",
"model_description": "SSD300-VGG-BN",
Expand Down Expand Up @@ -391,8 +391,8 @@
"ssd512_vgg_voc_int8": {
"config": "examples/torch/object_detection/configs/ssd512_vgg_voc_int8.json",
"reference": "ssd512_vgg_voc",
"target_ov": 80.19,
"target_pt": 80.11,
"target_ov": 80.18,
"target_pt": 80.14,
"metric_type": "Mean AP",
"resume": "ssd512_vgg_voc_int8.pth",
"batch": 32,
Expand All @@ -406,8 +406,8 @@
"ssd512_vgg_voc_magnitude_sparsity_int8": {
"config": "examples/torch/object_detection/configs/ssd512_vgg_voc_magnitude_sparsity_int8.json",
"reference": "ssd512_vgg_voc",
"target_ov": 79.98,
"target_pt": 79.7,
"target_ov": 79.96,
"target_pt": 79.76,
"metric_type": "Mean AP",
"resume": "ssd512_vgg_voc_magnitude_sparsity_int8.pth",
"batch": 32,
Expand Down Expand Up @@ -445,7 +445,7 @@
"unet_camvid_magnitude_sparsity_int8": {
"config": "examples/torch/semantic_segmentation/configs/unet_camvid_magnitude_sparsity_int8.json",
"reference": "unet_camvid",
"target_ov": 72.54,
"target_ov": 72.5,
"target_pt": 72.46,
"metric_type": "Mean IoU",
"resume": "unet_camvid_magnitude_sparsity_int8.pth",
Expand All @@ -467,8 +467,8 @@
"icnet_camvid_int8": {
"config": "examples/torch/semantic_segmentation/configs/icnet_camvid_int8.json",
"reference": "icnet_camvid",
"target_ov": 67.89,
"target_pt": 67.87,
"target_ov": 67.88,
"target_pt": 67.86,
"metric_type": "Mean IoU",
"resume": "icnet_camvid_int8.pth",
"model_description": "ICNet",
Expand All @@ -480,8 +480,8 @@
"icnet_camvid_magnitude_sparsity_int8": {
"config": "examples/torch/semantic_segmentation/configs/icnet_camvid_magnitude_sparsity_int8.json",
"reference": "icnet_camvid",
"target_ov": 67.16,
"target_pt": 67.16,
"target_ov": 67.2,
"target_pt": 67.17,
"metric_type": "Mean IoU",
"resume": "icnet_camvid_magnitude_sparsity_int8.pth",
"model_description": "ICNet",
Expand All @@ -504,7 +504,7 @@
"unet_mapillary_int8": {
"config": "examples/torch/semantic_segmentation/configs/unet_mapillary_int8.json",
"reference": "unet_mapillary",
"target_ov": 56.14,
"target_ov": 56.08,
"target_pt": 56.08,
"metric_type": "Mean IoU",
"resume": "unet_mapillary_int8.pth",
Expand All @@ -518,7 +518,7 @@
"unet_mapillary_magnitude_sparsity_int8": {
"config": "examples/torch/semantic_segmentation/configs/unet_mapillary_magnitude_sparsity_int8.json",
"reference": "unet_mapillary",
"target_ov": 55.76,
"target_ov": 55.72,
"target_pt": 55.7,
"metric_type": "Mean IoU",
"resume": "unet_mapillary_magnitude_sparsity_int8.pth",
Expand Down

0 comments on commit 489cc09

Please sign in to comment.