Skip to content

Commit

Permalink
[PT] Update targets for e2e (#2625)
Browse files Browse the repository at this point in the history 
### Changes

Set the value of `target_pt`, according to the average value obtained
from the build history.

### Reason for changes

Unstable results on CUDA.
  • Loading branch information
@AlexanderDokuchaev
AlexanderDokuchaev authored Apr 12, 2024
1 parent 755ef0f commit f878143
Showing 1 changed file with 17 additions and 17 deletions.
34 changes: 17 additions & 17 deletions tests/torch/sota_checkpoints_eval.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"config": "examples/torch/classification/configs/quantization/resnet50_imagenet_int8.json",
"reference": "resnet50_imagenet",
"target_ov": 76.39,
"target_pt": 76.45,
"target_pt": 76.41,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_int8.pth",
"model_description": "ResNet-50",
Expand All @@ -38,7 +38,7 @@
"config": "examples/torch/classification/configs/mixed_precision/resnet50_imagenet_mixed_int_hawq.json",
"reference": "resnet50_imagenet",
"target_ov": 75.61,
"target_pt": 75.86,
"target_pt": 75.94,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_int4_int8.pth",
"model_description": "ResNet-50",
Expand All @@ -50,7 +50,7 @@
"config": "examples/torch/classification/configs/sparsity_quantization/resnet50_imagenet_rb_sparsity_int8.json",
"reference": "resnet50_imagenet",
"target_ov": 75.39,
"target_pt": 75.42,
"target_pt": 75.41,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_rb_sparsity_int8.pth",
"model_description": "ResNet-50",
Expand All @@ -62,7 +62,7 @@
"config": "examples/torch/classification/configs/sparsity_quantization/resnet50_imagenet_rb_sparsity50_int8.json",
"reference": "resnet50_imagenet",
"target_ov": 75.44,
"target_pt": 75.47,
"target_pt": 75.5,
"metric_type": "Acc@1",
"resume": "resnet50_imagenet_rb_sparsity50_int8.pth",
"model_description": "ResNet-50",
Expand Down Expand Up @@ -92,7 +92,7 @@
"config": "examples/torch/classification/configs/quantization/inception_v3_imagenet_int8.json",
"reference": "inception_v3_imagenet",
"target_ov": 77.49,
"target_pt": 77.43,
"target_pt": 77.46,
"metric_type": "Acc@1",
"resume": "inception_v3_imagenet_int8.pth",
"model_description": "Inception V3",
Expand All @@ -105,7 +105,7 @@
"config": "examples/torch/classification/configs/sparsity_quantization/inception_v3_imagenet_rb_sparsity_int8.json",
"reference": "inception_v3_imagenet",
"target_ov": 76.34,
"target_pt": 76.32,
"target_pt": 76.34,
"metric_type": "Acc@1",
"resume": "inception_v3_imagenet_rb_sparsity_int8.pth",
"model_description": "Inception V3",
Expand All @@ -125,7 +125,7 @@
"config": "examples/torch/classification/configs/quantization/mobilenet_v2_imagenet_int8.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 71.01,
"target_pt": 71.3,
"target_pt": 71.22,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_int8.pth",
"model_description": "MobileNet V2",
Expand All @@ -139,7 +139,7 @@
"config": "examples/torch/classification/configs/quantization/mobilenet_v2_imagenet_int8_per_tensor.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 71.17,
"target_pt": 71.28,
"target_pt": 71.26,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_int8_per_tensor.pth",
"model_description": "MobileNet V2",
Expand All @@ -153,7 +153,7 @@
"config": "examples/torch/classification/configs/mixed_precision/mobilenet_v2_imagenet_mixed_int_hawq.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 70.52,
"target_pt": 70.57,
"target_pt": 70.68,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_int4_int8.pth",
"model_description": "MobileNet V2",
Expand All @@ -167,7 +167,7 @@
"config": "examples/torch/classification/configs/sparsity_quantization/mobilenet_v2_imagenet_rb_sparsity_int8.json",
"reference": "mobilenet_v2_imagenet",
"target_ov": 71.07,
"target_pt": 71.02,
"target_pt": 71.06,
"metric_type": "Acc@1",
"resume": "mobilenet_v2_imagenet_rb_sparsity_int8.pth",
"model_description": "MobileNet V2",
Expand Down Expand Up @@ -237,7 +237,7 @@
"config": "examples/torch/classification/configs/mixed_precision/squeezenet1_1_imagenet_mixed_int_hawq_old_eval.json",
"reference": "squeezenet1_1_imagenet",
"target_ov": 57.53,
"target_pt": 57.63,
"target_pt": 57.66,
"metric_type": "Acc@1",
"resume": "squeezenet1_1_imagenet_int4_int8.pth",
"model_description": "SqueezeNet V1.1",
Expand Down Expand Up @@ -325,7 +325,7 @@
"config": "examples/torch/object_detection/configs/ssd300_mobilenet_voc_magnitude_int8.json",
"reference": "ssd300_mobilenet_voc",
"target_ov": 63.01,
"target_pt": 62.97,
"target_pt": 62.99,
"metric_type": "Mean AP",
"resume": "ssd300_mobilenet_voc_magnitude_sparsity_int8.pth",
"model_description": "SSD300-MobileNet",
Expand Down Expand Up @@ -360,7 +360,7 @@
"config": "examples/torch/object_detection/configs/ssd300_vgg_voc_magnitude_sparsity_int8.json",
"reference": "ssd300_vgg_voc",
"target_ov": 77.46,
"target_pt": 77.67,
"target_pt": 77.66,
"metric_type": "Mean AP",
"resume": "ssd300_vgg_voc_magnitude_sparsity_int8.pth",
"model_description": "SSD300-VGG-BN",
Expand Down Expand Up @@ -392,7 +392,7 @@
"config": "examples/torch/object_detection/configs/ssd512_vgg_voc_int8.json",
"reference": "ssd512_vgg_voc",
"target_ov": 80.19,
"target_pt": 80.14,
"target_pt": 80.11,
"metric_type": "Mean AP",
"resume": "ssd512_vgg_voc_int8.pth",
"batch": 32,
Expand All @@ -407,7 +407,7 @@
"config": "examples/torch/object_detection/configs/ssd512_vgg_voc_magnitude_sparsity_int8.json",
"reference": "ssd512_vgg_voc",
"target_ov": 79.98,
"target_pt": 79.76,
"target_pt": 79.7,
"metric_type": "Mean AP",
"resume": "ssd512_vgg_voc_magnitude_sparsity_int8.pth",
"batch": 32,
Expand Down Expand Up @@ -468,7 +468,7 @@
"config": "examples/torch/semantic_segmentation/configs/icnet_camvid_int8.json",
"reference": "icnet_camvid",
"target_ov": 67.89,
"target_pt": 67.86,
"target_pt": 67.87,
"metric_type": "Mean IoU",
"resume": "icnet_camvid_int8.pth",
"model_description": "ICNet",
Expand All @@ -481,7 +481,7 @@
"config": "examples/torch/semantic_segmentation/configs/icnet_camvid_magnitude_sparsity_int8.json",
"reference": "icnet_camvid",
"target_ov": 67.16,
"target_pt": 67.17,
"target_pt": 67.16,
"metric_type": "Mean IoU",
"resume": "icnet_camvid_magnitude_sparsity_int8.pth",
"model_description": "ICNet",
Expand Down

0 comments on commit f878143

Please sign in to comment.