From e919e082f901a1f5766711c51e29e94dbf006e6b Mon Sep 17 00:00:00 2001 From: Alexander Dokuchaev Date: Thu, 12 Oct 2023 23:26:37 +0300 Subject: [PATCH] ov pt targets --- .../data/ac_configs/ssd300_mobilenet_voc.yml | 2 +- ..._mobilenet_voc_magnitude_sparsity_int8.yml | 2 +- .../torch/data/ac_configs/ssd300_vgg_voc.yml | 2 +- .../data/ac_configs/ssd300_vgg_voc_int8.yml | 2 +- ...ssd300_vgg_voc_magnitude_sparsity_int8.yml | 2 +- ...sd300_vgg_voc_pruning_geometric_median.yml | 2 +- .../torch/data/ac_configs/ssd512_vgg_voc.yml | 2 +- .../data/ac_configs/ssd512_vgg_voc_int8.yml | 2 +- ...ssd512_vgg_voc_magnitude_sparsity_int8.yml | 2 +- tests/torch/sota_checkpoints_eval.json | 149 ++++++++++++------ tests/torch/test_sota_checkpoints.py | 76 ++++----- 11 files changed, 137 insertions(+), 106 deletions(-) diff --git a/tests/torch/data/ac_configs/ssd300_mobilenet_voc.yml b/tests/torch/data/ac_configs/ssd300_mobilenet_voc.yml index c6e5d4a04e3..5246d3a372f 100644 --- a/tests/torch/data/ac_configs/ssd300_mobilenet_voc.yml +++ b/tests/torch/data/ac_configs/ssd300_mobilenet_voc.yml @@ -17,10 +17,10 @@ models: preprocessing: - type: resize size: 300 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd300_mobilenet_voc_magnitude_sparsity_int8.yml b/tests/torch/data/ac_configs/ssd300_mobilenet_voc_magnitude_sparsity_int8.yml index db46d4ed935..563efadf9fb 100644 --- a/tests/torch/data/ac_configs/ssd300_mobilenet_voc_magnitude_sparsity_int8.yml +++ b/tests/torch/data/ac_configs/ssd300_mobilenet_voc_magnitude_sparsity_int8.yml @@ -17,10 +17,10 @@ models: preprocessing: - type: resize size: 300 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd300_vgg_voc.yml b/tests/torch/data/ac_configs/ssd300_vgg_voc.yml index 09759704d77..e3268d5b817 100644 --- a/tests/torch/data/ac_configs/ssd300_vgg_voc.yml +++ b/tests/torch/data/ac_configs/ssd300_vgg_voc.yml @@ -17,10 +17,10 @@ models: preprocessing: - type: resize size: 300 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd300_vgg_voc_int8.yml b/tests/torch/data/ac_configs/ssd300_vgg_voc_int8.yml index 9dca5cf53c8..a7676d6f3f0 100644 --- a/tests/torch/data/ac_configs/ssd300_vgg_voc_int8.yml +++ b/tests/torch/data/ac_configs/ssd300_vgg_voc_int8.yml @@ -17,10 +17,10 @@ models: preprocessing: - type: resize size: 300 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd300_vgg_voc_magnitude_sparsity_int8.yml b/tests/torch/data/ac_configs/ssd300_vgg_voc_magnitude_sparsity_int8.yml index 14534eb0264..22972e3b2b3 100644 --- a/tests/torch/data/ac_configs/ssd300_vgg_voc_magnitude_sparsity_int8.yml +++ b/tests/torch/data/ac_configs/ssd300_vgg_voc_magnitude_sparsity_int8.yml @@ -17,10 +17,10 @@ models: preprocessing: - type: resize size: 300 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd300_vgg_voc_pruning_geometric_median.yml b/tests/torch/data/ac_configs/ssd300_vgg_voc_pruning_geometric_median.yml index 1f9ec60f4f2..5c324a02c7e 100644 --- a/tests/torch/data/ac_configs/ssd300_vgg_voc_pruning_geometric_median.yml +++ b/tests/torch/data/ac_configs/ssd300_vgg_voc_pruning_geometric_median.yml @@ -17,10 +17,10 @@ models: preprocessing: - type: resize size: 300 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd512_vgg_voc.yml b/tests/torch/data/ac_configs/ssd512_vgg_voc.yml index 7538ba7a89d..b0c4254288c 100644 --- a/tests/torch/data/ac_configs/ssd512_vgg_voc.yml +++ b/tests/torch/data/ac_configs/ssd512_vgg_voc.yml @@ -18,10 +18,10 @@ models: preprocessing: - type: resize size: 512 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd512_vgg_voc_int8.yml b/tests/torch/data/ac_configs/ssd512_vgg_voc_int8.yml index 55b86592d7d..2e385f3ec5b 100644 --- a/tests/torch/data/ac_configs/ssd512_vgg_voc_int8.yml +++ b/tests/torch/data/ac_configs/ssd512_vgg_voc_int8.yml @@ -18,10 +18,10 @@ models: preprocessing: - type: resize size: 512 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/data/ac_configs/ssd512_vgg_voc_magnitude_sparsity_int8.yml b/tests/torch/data/ac_configs/ssd512_vgg_voc_magnitude_sparsity_int8.yml index 71655b3c7c6..97f2e3e85f9 100644 --- a/tests/torch/data/ac_configs/ssd512_vgg_voc_magnitude_sparsity_int8.yml +++ b/tests/torch/data/ac_configs/ssd512_vgg_voc_magnitude_sparsity_int8.yml @@ -18,10 +18,10 @@ models: preprocessing: - type: resize size: 512 + - type: bgr_to_rgb - type: normalization mean: 123.675, 116.28, 103.53 std: 58.4795, 57.1429, 57.4713 - - type: bgr_to_rgb postprocessing: - type: resize_prediction_boxes metrics: diff --git a/tests/torch/sota_checkpoints_eval.json b/tests/torch/sota_checkpoints_eval.json index 34be1fb25b8..85d70c57087 100644 --- a/tests/torch/sota_checkpoints_eval.json +++ b/tests/torch/sota_checkpoints_eval.json @@ -3,14 +3,16 @@ "imagenet": { "resnet50_imagenet": { "config": "examples/torch/classification/configs/quantization/resnet50_imagenet.json", - "target": 76.16, + "target_ov": 76.16, + "target_pt": 76.15, "metric_type": "Acc@1", "model_description": "ResNet-50" }, "resnet50_imagenet_int8": { "config": "examples/torch/classification/configs/quantization/resnet50_imagenet_int8.json", "reference": "resnet50_imagenet", - "target": 76.39, + "target_ov": 76.35, + "target_pt": 76.45, "metric_type": "Acc@1", "resume": "resnet50_imagenet_int8.pth", "model_description": "ResNet-50", @@ -21,7 +23,8 @@ "resnet50_imagenet_int8_per_tensor": { "config": "examples/torch/classification/configs/quantization/resnet50_imagenet_int8_per_tensor.json", "reference": "resnet50_imagenet", - "target": 76.35, + "target_ov": 76.36, + "target_pt": 76.38, "metric_type": "Acc@1", "resume": "resnet50_imagenet_int8_per_tensor.pth", "model_description": "ResNet-50", @@ -32,7 +35,8 @@ "resnet50_imagenet_int4_int8": { "config": "examples/torch/classification/configs/mixed_precision/resnet50_imagenet_mixed_int_hawq.json", "reference": "resnet50_imagenet", - "target": 75.50, + "target_ov": 75.66, + "target_pt": 75.84, "metric_type": "Acc@1", "resume": "resnet50_imagenet_int4_int8.pth", "model_description": "ResNet-50", @@ -41,7 +45,8 @@ "resnet50_imagenet_rb_sparsity_int8": { "config": "examples/torch/classification/configs/sparsity_quantization/resnet50_imagenet_rb_sparsity_int8.json", "reference": "resnet50_imagenet", - "target": 75.39, + "target_ov": 75.66, + "target_pt": 75.42, "metric_type": "Acc@1", "resume": "resnet50_imagenet_rb_sparsity_int8.pth", "model_description": "ResNet-50", @@ -52,7 +57,8 @@ "resnet50_imagenet_rb_sparsity50_int8": { "config": "examples/torch/classification/configs/sparsity_quantization/resnet50_imagenet_rb_sparsity50_int8.json", "reference": "resnet50_imagenet", - "target": 75.44, + "target_ov": 75.46, + "target_pt": 75.47, "metric_type": "Acc@1", "resume": "resnet50_imagenet_rb_sparsity50_int8.pth", "model_description": "ResNet-50", @@ -63,7 +69,8 @@ "resnet50_imagenet_pruning_geometric_median": { "config": "examples/torch/classification/configs/pruning/resnet50_imagenet_pruning_geometric_median.json", "reference": "resnet50_imagenet", - "target": 75.57, + "target_ov": 75.57, + "target_pt": 75.57, "metric_type": "Acc@1", "resume": "resnet50_imagenet_pruning_geometric_median.pth", "model_description": "ResNet-50", @@ -71,7 +78,8 @@ }, "inception_v3_imagenet": { "config": "examples/torch/classification/configs/quantization/inception_v3_imagenet.json", - "target": 77.32, + "target_ov": 77.32, + "target_pt": 77.33, "metric_type": "Acc@1", "model_description": "Inception V3", "multiprocessing_distributed": true @@ -79,7 +87,8 @@ "inception_v3_imagenet_int8": { "config": "examples/torch/classification/configs/quantization/inception_v3_imagenet_int8.json", "reference": "inception_v3_imagenet", - "target": 77.45, + "target_ov": 77.44, + "target_pt": 77.45, "metric_type": "Acc@1", "resume": "inception_v3_imagenet_int8.pth", "model_description": "Inception V3", @@ -91,7 +100,8 @@ "inception_v3_imagenet_rb_sparsity_int8": { "config": "examples/torch/classification/configs/sparsity_quantization/inception_v3_imagenet_rb_sparsity_int8.json", "reference": "inception_v3_imagenet", - "target": 76.34, + "target_ov": 76.33, + "target_pt": 76.32, "metric_type": "Acc@1", "resume": "inception_v3_imagenet_rb_sparsity_int8.pth", "model_description": "Inception V3", @@ -102,14 +112,16 @@ }, "mobilenet_v2_imagenet": { "config": "examples/torch/classification/configs/quantization/mobilenet_v2_imagenet.json", - "target": 71.87, + "target_ov": 71.87, + "target_pt": 71.88, "metric_type": "Acc@1", "model_description": "MobileNet V2" }, "mobilenet_v2_imagenet_int8": { "config": "examples/torch/classification/configs/quantization/mobilenet_v2_imagenet_int8.json", "reference": "mobilenet_v2_imagenet", - "target": 71.01, + "target_ov": 71.01, + "target_pt": 71.24, "metric_type": "Acc@1", "resume": "mobilenet_v2_imagenet_int8.pth", "model_description": "MobileNet V2", @@ -120,7 +132,8 @@ "mobilenet_v2_imagenet_int8_per_tensor": { "config": "examples/torch/classification/configs/quantization/mobilenet_v2_imagenet_int8_per_tensor.json", "reference": "mobilenet_v2_imagenet", - "target": 71.17, + "target_ov": 71.08, + "target_pt": 71.18, "metric_type": "Acc@1", "resume": "mobilenet_v2_imagenet_int8_per_tensor.pth", "model_description": "MobileNet V2", @@ -131,7 +144,8 @@ "mobilenet_v2_imagenet_int4_int8": { "config": "examples/torch/classification/configs/mixed_precision/mobilenet_v2_imagenet_mixed_int_hawq.json", "reference": "mobilenet_v2_imagenet", - "target": 70.44, + "target_ov": 70.52, + "target_pt": 70.57, "metric_type": "Acc@1", "resume": "mobilenet_v2_imagenet_int4_int8.pth", "model_description": "MobileNet V2", @@ -142,7 +156,8 @@ "mobilenet_v2_imagenet_rb_sparsity_int8": { "config": "examples/torch/classification/configs/sparsity_quantization/mobilenet_v2_imagenet_rb_sparsity_int8.json", "reference": "mobilenet_v2_imagenet", - "target": 71.07, + "target_ov": 71.13, + "target_pt": 71.02, "metric_type": "Acc@1", "resume": "mobilenet_v2_imagenet_rb_sparsity_int8.pth", "model_description": "MobileNet V2", @@ -152,14 +167,16 @@ }, "mobilenet_v3_small_imagenet": { "config": "examples/torch/classification/configs/quantization/mobilenet_v3_small_imagenet.json", - "target": 67.66, + "target_ov": 67.68, + "target_pt": 67.66, "metric_type": "Acc@1", "model_description": "MobileNet V3 small" }, "mobilenet_v3_small_imagenet_int8": { "config": "examples/torch/classification/configs/quantization/mobilenet_v3_small_imagenet_int8.json", "reference": "mobilenet_v3_small_imagenet", - "target": 66.95, + "target_ov": 66.9, + "target_pt": 66.97, "metric_type": "Acc@1", "resume": "mobilenet_v3_small_imagenet_int8.pth", "model_description": "MobileNet V3 small", @@ -169,14 +186,16 @@ }, "squeezenet1_1_imagenet": { "config": "examples/torch/classification/configs/quantization/squeezenet1_1_imagenet.json", - "target": 58.19, + "target_ov": 58.19, + "target_pt": 58.17, "metric_type": "Acc@1", "model_description": "SqueezeNet V1.1" }, "squeezenet1_1_imagenet_int8": { "config": "examples/torch/classification/configs/quantization/squeezenet1_1_imagenet_int8.json", "reference": "squeezenet1_1_imagenet", - "target": 58.22, + "target_ov": 58.23, + "target_pt": 58.3, "metric_type": "Acc@1", "resume": "squeezenet1_1_imagenet_int8.pth", "model_description": "SqueezeNet V1.1", @@ -187,7 +206,8 @@ "squeezenet1_1_imagenet_int8_per_tensor": { "config": "examples/torch/classification/configs/quantization/squeezenet1_1_imagenet_int8_per_tensor.json", "reference": "squeezenet1_1_imagenet", - "target": 58.11, + "target_ov": 58.1, + "target_pt": 58.15, "metric_type": "Acc@1", "resume": "squeezenet1_1_imagenet_int8_per_tensor.pth", "model_description": "SqueezeNet V1.1", @@ -198,7 +218,8 @@ "squeezenet1_1_imagenet_int4_int8": { "config": "examples/torch/classification/configs/mixed_precision/squeezenet1_1_imagenet_mixed_int_hawq_old_eval.json", "reference": "squeezenet1_1_imagenet", - "target": 57.57, + "target_ov": 57.62, + "target_pt": 57.59, "metric_type": "Acc@1", "resume": "squeezenet1_1_imagenet_int4_int8.pth", "model_description": "SqueezeNet V1.1", @@ -208,25 +229,28 @@ }, "resnet18_imagenet": { "config": "examples/torch/classification/configs/binarization/resnet18_imagenet.json", - "target": 69.77, + "target_ov": 69.76, + "target_pt": 69.76, "metric_type": "Acc@1", "model_description": "ResNet-18" }, "resnet18_imagenet_binarization_xnor": { "config": "examples/torch/classification/configs/binarization/resnet18_imagenet_binarization_xnor.json", "reference": "resnet18_imagenet", - "target": 61.82, + "target_ov": 61.59, + "target_pt": 61.74, "metric_type": "Acc@1", "resume": "resnet18_imagenet_binarization_xnor.pth", "model_description": "ResNet-18", "compression_description": "XNOR (weights), scale/threshold (activations)", - "diff_fp32_min": -8, + "diff_fp32_min": -8.1, "diff_fp32_max": 0.1 }, "resnet18_imagenet_binarization_dorefa": { "config": "examples/torch/classification/configs/binarization/resnet18_imagenet_binarization_dorefa.json", "reference": "resnet18_imagenet", - "target": 61.49, + "target_ov": null, + "target_pt": 61.49, "metric_type": "Acc@1", "resume": "resnet18_imagenet_binarization_dorefa.pth", "model_description": "ResNet-18", @@ -238,7 +262,8 @@ "resnet18_imagenet_pruning_magnitude": { "config": "examples/torch/classification/configs/pruning/resnet18_imagenet_pruning_magnitude.json", "reference": "resnet18_imagenet", - "target": 69.26, + "target_ov": 69.26, + "target_pt": 69.27, "metric_type": "Acc@1", "resume": "resnet18_imagenet_pruning_magnitude.pth", "model_description": "ResNet-18", @@ -247,7 +272,8 @@ "resnet18_imagenet_pruning_geometric_median": { "config": "examples/torch/classification/configs/pruning/resnet18_imagenet_pruning_geometric_median.json", "reference": "resnet18_imagenet", - "target": 69.30, + "target_ov": 69.76, + "target_pt": 69.31, "metric_type": "Acc@1", "resume": "resnet18_imagenet_pruning_geometric_median.pth", "model_description": "ResNet-18", @@ -255,14 +281,16 @@ }, "resnet34_imagenet": { "config": "examples/torch/classification/configs/pruning/resnet34_imagenet.json", - "target": 73.3, + "target_ov": 73.31, + "target_pt": 73.29, "metric_type": "Acc@1", "model_description": "ResNet-34" }, "resnet34_imagenet_pruning_geometric_median_kd": { "config": "examples/torch/classification/configs/pruning/resnet34_imagenet_pruning_geometric_median_kd.json", "reference": "resnet34_imagenet", - "target": 73.12, + "target_ov": 73.12, + "target_pt": 73.12, "metric_type": "Acc@1", "resume": "resnet34_imagenet_pruning_geometric_median_kd.pth", "model_description": "ResNet-34", @@ -270,14 +298,16 @@ }, "googlenet_imagenet": { "config": "examples/torch/classification/configs/pruning/googlenet_imagenet.json", - "target": 69.77, + "target_ov": 69.77, + "target_pt": 69.78, "metric_type": "Acc@1", "model_description": "GoogLeNet" }, "googlenet_imagenet_pruning_geometric_median": { "config": "examples/torch/classification/configs/pruning/googlenet_imagenet_pruning_geometric_median.json", "reference": "googlenet_imagenet", - "target": 69.45, + "target_ov": 69.45, + "target_pt": 69.46, "metric_type": "Acc@1", "resume": "googlenet_imagenet_pruning_geometric_median.pth", "model_description": "GoogLeNet", @@ -289,7 +319,8 @@ "voc": { "ssd300_mobilenet_voc": { "config": "examples/torch/object_detection/configs/ssd300_mobilenet_voc.json", - "target": 62.23, + "target_ov": 62.27, + "target_pt": 62.24, "metric_type": "Mean AP", "resume": "ssd300_mobilenet_voc.pth", "batch": 120, @@ -298,7 +329,8 @@ "ssd300_mobilenet_voc_magnitude_sparsity_int8": { "config": "examples/torch/object_detection/configs/ssd300_mobilenet_voc_magnitude_int8.json", "reference": "ssd300_mobilenet_voc", - "target": 62.95, + "target_ov": 62.97, + "target_pt": 62.97, "metric_type": "Mean AP", "resume": "ssd300_mobilenet_voc_magnitude_sparsity_int8.pth", "model_description": "SSD300-MobileNet", @@ -309,7 +341,8 @@ }, "ssd300_vgg_voc": { "config": "examples/torch/object_detection/configs/ssd300_vgg_voc.json", - "target": 78.28, + "target_ov": 78.03, + "target_pt": 78.28, "metric_type": "Mean AP", "resume": "ssd300_vgg_voc.pth", "batch": 120, @@ -319,7 +352,8 @@ "ssd300_vgg_voc_int8": { "config": "examples/torch/object_detection/configs/ssd300_vgg_voc_int8.json", "reference": "ssd300_vgg_voc", - "target": 77.81, + "target_ov": 77.99, + "target_pt": 77.89, "metric_type": "Mean AP", "resume": "ssd300_vgg_voc_int8.pth", "model_description": "SSD300-VGG-BN", @@ -330,7 +364,8 @@ "ssd300_vgg_voc_magnitude_sparsity_int8": { "config": "examples/torch/object_detection/configs/ssd300_vgg_voc_magnitude_sparsity_int8.json", "reference": "ssd300_vgg_voc", - "target": 77.66, + "target_ov": 77.65, + "target_pt": 77.67, "metric_type": "Mean AP", "resume": "ssd300_vgg_voc_magnitude_sparsity_int8.pth", "model_description": "SSD300-VGG-BN", @@ -340,7 +375,8 @@ "ssd300_vgg_voc_pruning_geometric_median": { "config": "examples/torch/object_detection/configs/ssd300_vgg_voc_pruning_geometric_median.json", "reference": "ssd300_vgg_voc", - "target": 78.35, + "target_ov": 77.98, + "target_pt": 78.35, "metric_type": "Mean AP", "resume": "ssd300_vgg_voc_pruning_geometric_median.pth", "batch": 32, @@ -350,7 +386,8 @@ }, "ssd512_vgg_voc": { "config": "examples/torch/object_detection/configs/ssd512_vgg_voc.json", - "target": 80.26, + "target_ov": 80.59, + "target_pt": 80.26, "metric_type": "Mean AP", "resume": "ssd512_vgg_voc.pth", "batch": 32, @@ -360,7 +397,8 @@ "ssd512_vgg_voc_int8": { "config": "examples/torch/object_detection/configs/ssd512_vgg_voc_int8.json", "reference": "ssd512_vgg_voc", - "target": 80.04, + "target_ov": 80.13, + "target_pt": 80.09, "metric_type": "Mean AP", "resume": "ssd512_vgg_voc_int8.pth", "batch": 32, @@ -373,7 +411,8 @@ "ssd512_vgg_voc_magnitude_sparsity_int8": { "config": "examples/torch/object_detection/configs/ssd512_vgg_voc_magnitude_sparsity_int8.json", "reference": "ssd512_vgg_voc", - "target": 79.68, + "target_ov": 79.95, + "target_pt": 79.76, "metric_type": "Mean AP", "resume": "ssd512_vgg_voc_magnitude_sparsity_int8.pth", "batch": 32, @@ -389,7 +428,8 @@ "camvid": { "unet_camvid": { "config": "examples/torch/semantic_segmentation/configs/unet_camvid.json", - "target": 71.93, + "target_ov": 71.9, + "target_pt": 71.95, "metric_type": "Mean IoU", "resume": "unet_camvid.pth", "model_description": "UNet", @@ -398,7 +438,8 @@ "unet_camvid_int8": { "config": "examples/torch/semantic_segmentation/configs/unet_camvid_int8.json", "reference": "unet_camvid", - "target": 71.89, + "target_ov": 71.88, + "target_pt": 71.9, "metric_type": "Mean IoU", "resume": "unet_camvid_int8.pth", "model_description": "UNet", @@ -411,7 +452,8 @@ "unet_camvid_magnitude_sparsity_int8": { "config": "examples/torch/semantic_segmentation/configs/unet_camvid_magnitude_sparsity_int8.json", "reference": "unet_camvid", - "target": 72.46, + "target_ov": 72.55, + "target_pt": 72.46, "metric_type": "Mean IoU", "resume": "unet_camvid_magnitude_sparsity_int8.pth", "model_description": "UNet", @@ -422,7 +464,8 @@ }, "icnet_camvid": { "config": "examples/torch/semantic_segmentation/configs/icnet_camvid.json", - "target": 67.89, + "target_ov": 67.87, + "target_pt": 67.89, "metric_type": "Mean IoU", "resume": "icnet_camvid.pth", "model_description": "ICNet", @@ -431,7 +474,8 @@ "icnet_camvid_int8": { "config": "examples/torch/semantic_segmentation/configs/icnet_camvid_int8.json", "reference": "icnet_camvid", - "target": 67.89, + "target_ov": 67.91, + "target_pt": 67.86, "metric_type": "Mean IoU", "resume": "icnet_camvid_int8.pth", "model_description": "ICNet", @@ -443,7 +487,8 @@ "icnet_camvid_magnitude_sparsity_int8": { "config": "examples/torch/semantic_segmentation/configs/icnet_camvid_magnitude_sparsity_int8.json", "reference": "icnet_camvid", - "target": 67.16, + "target_ov": 67.18, + "target_pt": 67.17, "metric_type": "Mean IoU", "resume": "icnet_camvid_magnitude_sparsity_int8.pth", "model_description": "ICNet", @@ -456,7 +501,8 @@ "mapillary_vistas": { "unet_mapillary": { "config": "examples/torch/semantic_segmentation/configs/unet_mapillary.json", - "target": 56.24, + "target_ov": 56.24, + "target_pt": 56.24, "metric_type": "Mean IoU", "resume": "unet_mapillary.pth", "model_description": "UNet", @@ -465,7 +511,8 @@ "unet_mapillary_int8": { "config": "examples/torch/semantic_segmentation/configs/unet_mapillary_int8.json", "reference": "unet_mapillary", - "target": 56.09, + "target_ov": 56.09, + "target_pt": 56.08, "metric_type": "Mean IoU", "resume": "unet_mapillary_int8.pth", "model_description": "UNet", @@ -477,7 +524,8 @@ "unet_mapillary_magnitude_sparsity_int8": { "config": "examples/torch/semantic_segmentation/configs/unet_mapillary_magnitude_sparsity_int8.json", "reference": "unet_mapillary", - "target": 55.69, + "target_ov": 55.72, + "target_pt": 55.7, "metric_type": "Mean IoU", "resume": "unet_mapillary_magnitude_sparsity_int8.pth", "model_description": "UNet", @@ -490,7 +538,8 @@ "unet_mapillary_pruning_geometric_median": { "config": "examples/torch/semantic_segmentation/configs/unet_mapillary_pruning_geometric_median.json", "reference": "unet_mapillary", - "target": 55.64, + "target_ov": 55.64, + "target_pt": 55.64, "metric_type": "Mean IoU", "resume": "unet_mapillary_pruning_geometric_median.pth", "model_description": "UNet", diff --git a/tests/torch/test_sota_checkpoints.py b/tests/torch/test_sota_checkpoints.py index afd74ab664a..3ea069fb413 100644 --- a/tests/torch/test_sota_checkpoints.py +++ b/tests/torch/test_sota_checkpoints.py @@ -39,7 +39,8 @@ class EvalRunParamsStruct: config_name: str reference: Optional[str] - expected: float + target_ov: float + target_pt: float metric_type: str dataset_name: str sample_type: str @@ -103,11 +104,14 @@ def read_reference_file(ref_path: Path) -> List[EvalRunParamsStruct]: for dataset_name in datasets: model_dict = datasets[dataset_name] for model_name, sample_dict in model_dict.items(): + if "target_pt" not in sample_dict: + continue param_list.append( EvalRunParamsStruct( config_name=sample_dict["config"], reference=sample_dict.get("reference", None), - expected=sample_dict["target"], + target_pt=sample_dict["target_pt"], + target_ov=sample_dict["target_ov"], metric_type=sample_dict["metric_type"], dataset_name=dataset_name, sample_type=sample_type_, @@ -209,7 +213,7 @@ def setup_class(self): self.ref_fp32_dict = OrderedDict() for run_param in EVAL_TEST_STRUCT: if run_param.reference is None: - self.ref_fp32_dict[run_param.model_name] = run_param.expected + self.ref_fp32_dict[run_param.model_name] = run_param.target_ov @pytest.fixture(params=EVAL_TEST_STRUCT, ids=idfn) def eval_run_param(self, request): @@ -251,24 +255,19 @@ def get_reference_fp32_metric(self, metrics_dump_path: Path, reference_name: str @staticmethod def threshold_check( - diff_target, - diff_fp32, - diff_target_min, - diff_target_max, - diff_fp32_min, - diff_fp32_max, + diff_target: float, diff_fp32: Optional[float], param: EvalRunParamsStruct ) -> Tuple[bool, List[str]]: err_msgs = [] - - if diff_target < diff_target_min or diff_target > diff_target_max: + if diff_target < param.diff_target_min or diff_target > param.diff_target_max: err_msgs.append( - f"Target diff is not within thresholds: {diff_target_min} < {diff_target} < {diff_target_max}" + "Target diff is not within thresholds: " + + f"{param.diff_target_min} < {diff_target} < {param.diff_target_max}" ) - if diff_fp32 is not None: - if diff_fp32 < diff_fp32_min or diff_fp32 > diff_fp32_max: - err_msgs.append(f"FP32 diff is not within thresholds: {diff_fp32_min} < {diff_fp32} < {diff_fp32_max}") - + if diff_fp32 < param.diff_fp32_min or diff_fp32 > param.diff_fp32_max: + err_msgs.append( + f"FP32 diff is not within thresholds: {param.diff_fp32_min} < {diff_fp32} < {param.diff_fp32_max}" + ) if err_msgs: return ";".join(err_msgs) return None @@ -318,35 +317,28 @@ def test_eval(self, sota_checkpoints_dir, sota_data_dir, eval_run_param: EvalRun diff_target = None diff_fp32 = None if not is_ok: + status = f"exit_code: {exit_code}" result_info = ResultInfo( model_name=eval_run_param.model_name, backend=PYTORCH, - status=f"exit_code: {exit_code}", + status=status, ) add_test_result(result_info) - pytest.fail(f"exit_code: {exit_code}") + pytest.fail(status) metric_value = self.read_metric(metrics_dump_file_path) - fp32_metric = self.get_reference_fp32_metric(pytest.metrics_dump_path, eval_run_param.reference) - diff_target = round((metric_value - eval_run_param.expected), 2) + diff_target = round((metric_value - eval_run_param.target_pt), 2) if fp32_metric: diff_fp32 = round((metric_value - fp32_metric), 2) - threshold_errors = self.threshold_check( - diff_target=diff_target, - diff_fp32=diff_fp32, - diff_target_min=eval_run_param.diff_target_min, - diff_target_max=eval_run_param.diff_target_max, - diff_fp32_min=eval_run_param.diff_fp32_min, - diff_fp32_max=eval_run_param.diff_fp32_max, - ) + threshold_errors = self.threshold_check(diff_target=diff_target, diff_fp32=diff_fp32, param=eval_run_param) result_info = ResultInfo( model_name=eval_run_param.model_name, backend=PYTORCH, metric_type=eval_run_param.metric_type, - expected=eval_run_param.expected, + expected=eval_run_param.target_pt, measured=metric_value, diff_fp32=diff_fp32, diff_target=diff_target, @@ -414,49 +406,41 @@ def test_openvino_eval(self, eval_run_param: EvalRunParamsStruct, ov_data_dir, o pytest.fail(f"{ir_model_path} does not exists") ac_yml_path = config_folder / f"{eval_run_param.model_name}.yml" - report_csv_path = pytest.metrics_dump_path / f"{eval_run_param.model_name}.csv" # Ensure that report file does not exists report_csv_path.unlink(missing_ok=True) cmd = self.generate_accuracy_check_cmd(ac_yml_path, ov_data_dir, ir_model_path.parent, report_csv_path) - runner = Command(cmd, cwd=PROJECT_ROOT, env=self.get_env()) exit_code = runner.run(assert_returncode_zero=False) if exit_code: + status = f"Accuracy checker return code: {exit_code}" add_test_result( ResultInfo( model_name=eval_run_param.model_name, backend=OPENVINO, - status=f"Accuracy checker return code: {exit_code}", + status=status, ) ) - pytest.fail(f"Accuracy checker return code: {exit_code}") + pytest.fail(status) metric_value = self.get_metric_from_ac_csv(report_csv_path) - fp32_metric = self.get_reference_fp32_metric(pytest.metrics_dump_path, eval_run_param.reference) - diff_target = round((metric_value - eval_run_param.expected), 2) + diff_target = round((metric_value - eval_run_param.target_ov), 2) diff_fp32 = None if fp32_metric: diff_fp32 = round((metric_value - fp32_metric), 2) - threshold_errors = self.threshold_check( - diff_target=diff_target, - diff_fp32=diff_fp32, - diff_target_min=eval_run_param.diff_target_min, - diff_target_max=eval_run_param.diff_target_max, - diff_fp32_min=eval_run_param.diff_fp32_min, - diff_fp32_max=eval_run_param.diff_fp32_max, - ) + threshold_errors = self.threshold_check(diff_target=diff_target, diff_fp32=diff_fp32, param=eval_run_param) + result_info = ResultInfo( model_name=eval_run_param.model_name, backend=OPENVINO, metric_type=eval_run_param.metric_type, - expected=eval_run_param.expected, + expected=eval_run_param.target_ov, measured=metric_value, diff_fp32=diff_fp32, diff_target=diff_target, @@ -491,15 +475,13 @@ def test_train(self, eval_run_param: EvalRunParamsStruct, cuda_ip, sota_data_dir cuda_ip=cuda_ip, weights_path=weights_path, ) - - fp32_metric = self.ref_fp32_dict[eval_run_param.reference] - runner = Command(cmd, cwd=PROJECT_ROOT, env=self.get_env()) exit_code = runner.run(assert_returncode_zero=False) is_ok = exit_code == 0 and metrics_dump_file_path.exists() err_msg = None if is_ok: + fp32_metric = self.ref_fp32_dict[eval_run_param.reference] metric_value = self.read_metric(str(metrics_dump_file_path)) diff_fp32 = round((metric_value - fp32_metric), 2) if -1 < diff_fp32: