From 548ee58f3a7cac11870b1d3a9da8352c6c6c6948 Mon Sep 17 00:00:00 2001 From: Vinnam Kim Date: Mon, 30 May 2022 11:25:56 +0900 Subject: [PATCH] Exp public ptq api - ONNX Vision classification (#1188) * [ONNX PTQ API] Auto-infer input shape Signed-off-by: Kim, Vinnam * Add ONNX Dockerfile Signed-off-by: Kim, Vinnam * Add onnx-models' classification model configs Signed-off-by: Kim, Vinnam * Update ONNX Dockerfile Signed-off-by: Kim, Vinnam * Add batchsize to examples/experimental/onnx/run_ptq_onnx_models.sh Signed-off-by: Kim, Vinnam * Force PTQ-API to use CPUExecutionProvider - OpenVINOExecutionProvider cannot PTQ many models (e.g. ONNX models ResNet-50) Signed-off-by: Kim, Vinnam * Add README.md Signed-off-by: Kim, Vinnam * Rollback examples/experimental/onnx/classification/ac_configs/resnet50.yml Signed-off-by: Kim, Vinnam * Remove self.sess_options Signed-off-by: Kim, Vinnam * Fix tests/onnx failures Signed-off-by: Kim, Vinnam --- .dockerignore | 12 ++ .gitignore | 8 +- docker/onnx/openvinoep/Dockerfile | 38 ++++++ docker/onnx/openvinoep/build.sh | 15 +++ examples/experimental/onnx/README.md | 121 +++++++++++++++++ .../onnx_models_configs/bvlcalexnet-12.yml | 35 +++++ .../onnx_models_configs/caffenet-12.yml | 34 +++++ .../onnx_models_configs/densenet-12.yml | 37 ++++++ .../efficientnet-lite4-11.yml | 38 ++++++ .../onnx_models_configs/googlenet-12.yml | 35 +++++ .../onnx_models_configs/inception-v1-12.yml | 35 +++++ .../onnx_models_configs/inception-v2-9.yml | 35 +++++ .../onnx_models_configs/mobilenetv2-12.yml | 37 ++++++ .../onnx_models_configs/resnet50-v1-12.yml | 37 ++++++ .../onnx_models_configs/resnet50-v2-7.yml | 37 ++++++ .../onnx_models_configs/shufflenet-9.yml | 37 ++++++ .../onnx_models_configs/shufflenet-v2-12.yml | 37 ++++++ .../onnx_models_configs/squeezenet1.0-12.yml | 35 +++++ .../onnx_models_configs/vgg16-12.yml | 37 ++++++ .../onnx_models_configs/zfnet512-12.yml | 35 +++++ .../classification/onnx_ptq_classification.py | 33 +++-- examples/experimental/onnx/requirements.txt | 3 +- examples/experimental/onnx/run_ptq.py | 122 ++++++++++++++++++ .../experimental/onnx/run_ptq_onnx_models.sh | 54 ++++++++ .../onnx/datasets/imagenet_dataset.py | 71 +++++++--- nncf/experimental/onnx/engine.py | 7 +- nncf/experimental/onnx/model_normalizer.py | 2 + .../test_classification_sanity_sample.py | 3 +- 28 files changed, 1000 insertions(+), 30 deletions(-) create mode 100644 .dockerignore create mode 100644 docker/onnx/openvinoep/Dockerfile create mode 100755 docker/onnx/openvinoep/build.sh create mode 100644 examples/experimental/onnx/README.md create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/bvlcalexnet-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/caffenet-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/densenet-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/efficientnet-lite4-11.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/googlenet-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/inception-v1-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/inception-v2-9.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/mobilenetv2-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/resnet50-v1-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/resnet50-v2-7.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/shufflenet-9.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/shufflenet-v2-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/squeezenet1.0-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/vgg16-12.yml create mode 100644 examples/experimental/onnx/classification/onnx_models_configs/zfnet512-12.yml create mode 100644 examples/experimental/onnx/run_ptq.py create mode 100755 examples/experimental/onnx/run_ptq_onnx_models.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..b5bcb538adb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git/ + +build +dist +*.egg-info +*.egg/ +*.pyc +*.swp + +.tox +.coverage +__pycache__ diff --git a/.gitignore b/.gitignore index 23b81e5a06b..51fa941779c 100644 --- a/.gitignore +++ b/.gitignore @@ -108,4 +108,10 @@ ENV/ *.tar # object detection eval results -examples/torch/object_detection/eval/ \ No newline at end of file +examples/torch/object_detection/eval/ + +# VSCode +.vscode/ + +# NNCF debug +nncf_debug/ diff --git a/docker/onnx/openvinoep/Dockerfile b/docker/onnx/openvinoep/Dockerfile new file mode 100644 index 00000000000..69accc53379 --- /dev/null +++ b/docker/onnx/openvinoep/Dockerfile @@ -0,0 +1,38 @@ +FROM openvino/onnxruntime_ep_ubuntu18:latest +USER root + +ARG PIP_EXTRA_INDEX_URL +ARG PIP_TRUSTED_HOST +ARG http_proxy +ARG https_proxy +ARG no_proxy + +RUN echo "PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}" +RUN echo "PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}" +RUN echo "http_proxy=${http_proxy}" +RUN echo "https_proxy=${https_proxy}" +RUN echo "no_proxy=${no_proxy}" + +RUN apt update && apt install -y git \ + && rm -rf /var/lib/apt/lists/* + +RUN python3 -m pip install -U pip wheel + +WORKDIR /opt/nncf_ws +ADD nncf/experimental/onnx/requirements.txt onnx_requirements.txt +RUN python3 -m pip install -r onnx_requirements.txt + +ADD examples/experimental/onnx/requirements.txt example_requirements.txt +RUN python3 -m pip install -r example_requirements.txt + +ADD nncf ./nncf +ADD setup.py ./ +ADD README.md ./ + +RUN python3 -m pip install -e . + +RUN ln -s /usr/bin/python3 /usr/bin/python + +USER onnxruntimedev +WORKDIR /home/onnxruntimedev +ADD examples/experimental/onnx ./examples diff --git a/docker/onnx/openvinoep/build.sh b/docker/onnx/openvinoep/build.sh new file mode 100755 index 00000000000..c461976a82c --- /dev/null +++ b/docker/onnx/openvinoep/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(dirname "$0") +WORK_DIR="${SCRIPT_DIR}/../../../" + +cd $WORK_DIR && echo "WORK_DIR=$PWD" + +docker build -t onnx_ptq_experimental:latest \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + --build-arg no_proxy=$no_proxy \ + --build-arg PIP_EXTRA_INDEX_URL=$PIP_EXTRA_INDEX_URL \ + --build-arg PIP_TRUSTED_HOST=$PIP_TRUSTED_HOST \ + -f docker/onnx/openvinoep/Dockerfile \ + . diff --git a/examples/experimental/onnx/README.md b/examples/experimental/onnx/README.md new file mode 100644 index 00000000000..6b07f0e409f --- /dev/null +++ b/examples/experimental/onnx/README.md @@ -0,0 +1,121 @@ +# Post-Training Quantization (PTQ) using ONNXRuntime + +This examples shows how to quantize ONNX formated NN model (FP32) into the quantized NN model (INT8) using NNCF PTQ API with ONNXRuntime framework. + +## Docker image build + +You should make an environment including [ONNXRuntime](https://onnxruntime.ai/docs) with [OpenVINOExecutionProvider](https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html). We officially support `onnxruntime==1.11.0` and `openvino==2022.1.0`. You can use use a docker image build script we provided in `./docker/onnx/openvinoep/build.sh` to configure the environment easily. + +```bash +# Build +$ ./docker/onnx/openvinoep/build.sh +... +Successfully tagged onnx_ptq_experimental:latest + +# Check image +$ docker images | grep onnx_ptq_experimental:latest +``` + +## Run NNCF PTQ for ONNXRuntime for your model + +Please refer to guides: + +1. [Classification models](classification/README.md) +2. [Semantic segmenantation models](semantic_segmentation/README.md) + +## Benchmark for ONNX Model Zoo + +[ONNX Model Zoo](https://github.com/onnx/models) provides is an open standard format for popular deep NN models with their pretrained weights. In this examples, we will quantize ONNX Model ZOO models. After quantization, model accuracy and model latency are compared between the original model (FP32) and quantized model (INT8). + +## Benchmark for ONNX Models: Vision - Classification + +### Prepare dataset + +Because we use [OpenVINO™ Accuracy Checker](https://github.com/openvinotoolkit/open_model_zoo/tree/master/tools/accuracy_checker) tool, you should prepare ILSVRC2012 validation dataset by following the [dataset preparation guide](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/data/datasets.md#imagenet). After preparation, your dataset directory will be: + +``` +DATASET_DIR ++-- ILSVRC2012_img_val +| +-- ILSVRC2012_val_00000001.JPEG +| +-- ILSVRC2012_val_00000002.JPEG +| +-- ILSVRC2012_val_00000003.JPEG +| +-- ... ++-- val.txt +``` + +### Prepare models + +You can download models from [ONNX Model Zoo - Image Classification](https://github.com/onnx/models#image_classification). +In this example, you have to prepare 15 classification models. + +1. [bvlcalexnet-12](https://github.com/onnx/models/blob/main/vision/classification/alexnet/model/bvlcalexnet-12.onnx) +2. [caffenet-12](https://github.com/onnx/models/blob/main/vision/classification/caffenet/model/caffenet-12.onnx) +3. [densenet-12](https://github.com/onnx/models/blob/main/vision/classification/densenet-121/model/densenet-12.onnx) +4. [efficientnet-lite4-11](https://github.com/onnx/models/blob/main/vision/classification/efficientnet-lite4/model/efficientnet-lite4-11.onnx) +5. [googlenet-12](https://github.com/onnx/models/blob/main/vision/classification/inception_and_googlenet/googlenet/model/googlenet-12.onnx) +6. [inception-v1-12](https://github.com/onnx/models/blob/main/vision/classification/inception_and_googlenet/inception_v1/model/inception-v1-12.onnx) +7. [inception-v2-9](https://github.com/onnx/models/blob/main/vision/classification/inception_and_googlenet/inception_v2/model/inception-v2-9.onnx) +8. [mobilenetv2-12](https://github.com/onnx/models/blob/main/vision/classification/mobilenet/model/mobilenetv2-12.onnx) +9. [resnet50-v1-12](https://github.com/onnx/models/blob/main/vision/classification/resnet/model/resnet50-v1-12.onnx) +10. [resnet50-v2-7](https://github.com/onnx/models/blob/main/vision/classification/resnet/model/resnet50-v2-7.onnx) +11. [shufflenet-9](https://github.com/onnx/models/blob/main/vision/classification/shufflenet/model/shufflenet-9.onnx) +12. [shufflenet-v2-12](https://github.com/onnx/models/blob/main/vision/classification/shufflenet/model/shufflenet-v2-12.onnx) +13. [squeezenet1.0-12](https://github.com/onnx/models/blob/main/vision/classification/squeezenet/model/squeezenet1.0-12.onnx) +14. [vgg16-12](https://github.com/onnx/models/blob/main/vision/classification/vgg/model/vgg16-12.onnx) +15. [zfnet512-12](https://github.com/onnx/models/blob/main/vision/classification/zfnet-512/model/zfnet512-12.onnx) + +All downloaded models are located in the same directory. `MODELS_DIR` should be the following structure. +``` +MODELS_DIR ++-- bvlcalexnet-12.onnx ++-- caffenet-12.onnx ++-- densenet-12.onnx ++-- ... +(Total 15 onnx files) +``` + +### Prepare docker image + +Please refer to [Docker image build](#docker-image-build) section. + +### Run benchmark + +```bash +(host) $ docker run \ + -it --rm --name onnx_ptq \ + -v :/omz_data \ + -v :/onnx-models \ + -v :/output \ + onnx_ptq_experimental:latest +(container) $ ./examples/run_ptq_onnx_models.sh /onnx-models /output $NUMBER_OF_SAMPLES +``` + +`NUMBER_OF_SAMPLES` is an integer value which is the number of samples required for PTQ parameter calibrations and accuracy checks. For examples, to run with `NUMBER_OF_SAMPLES=500`, you can command as follows. + +```bash +(container) $ ./examples/run_ptq_onnx_models.sh /onnx-models /output 500 +``` + +After benchmark is done, outputs are located in `/output` which is a mounted directory from the host path ``. + +### Results + +| name | FP32 latency (ms) | INT8 latency (ms) | Latency diff. (FP32/INT8) | FP32 accuracy (%) | INT8 accuracy (%) | Accuracy diff. (%) | +|:----------------------|-------------------:|--------------------:|---------------:|--------------------:|---------------------:|----------------:| +| bvlcalexnet-12 | 26.12 | 5.56 | 4.70 | 50.20 | 49.80 | 0.40 | +| caffenet-12 | 26.11 | 5.17 | 5.05 | 54.40 | 54.40 | 0.00 | +| densenet-12 | 30.39 | 38.85 | 0.78 | 59.00 | nan | nan | +| efficientnet-lite4-11 | 18.44 | 6.45 | 2.86 | 77.80 | 77.60 | 0.20 | +| googlenet-12 | 15.31 | 9.26 | 1.65 | 68.40 | 67.60 | 0.80 | +| inception-v1-12 | 14.02 | nan | nan | 67.20 | nan | nan | +| inception-v2-9 | 18.75 | 10.02 | 1.87 | 0.00 | nan | nan | +| mobilenetv2-12 | 5.58 | 1.86 | 3.00 | 71.80 | 71.20 | 0.60 | +| resnet50-v1-12 | 32.11 | 14.30 | 2.25 | 73.60 | 72.80 | 0.80 | +| resnet50-v2-7 | 41.09 | 11.05 | 3.72 | 73.80 | 74.00 | -0.20 | +| shufflenet-9 | 4.58 | nan | nan | 46.60 | 0.00 | 46.60 | +| shufflenet-v2-12 | 3.40 | nan | nan | 68.60 | nan | nan | +| squeezenet1.0-12 | 3.76 | 1.85 | 2.03 | 53.20 | 53.60 | -0.40 | +| vgg16-12 | 159.44 | 38.75 | 4.11 | 70.80 | 70.60 | 0.20 | +| zfnet512-12 | 43.68 | 14.46 | 3.02 | 58.60 | 59.00 | -0.40 | + +* `nan` means that NNCF PTQ API failed to generate proper quantized onnx model. We are working on these defects. diff --git a/examples/experimental/onnx/classification/onnx_models_configs/bvlcalexnet-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/bvlcalexnet-12.yml new file mode 100644 index 00000000000..13d7f22aea5 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/bvlcalexnet-12.yml @@ -0,0 +1,35 @@ +models: +- name: bvlcalexnet-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: rgb_to_bgr + + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # BGR format + - type: normalization + mean: (103.939, 116.779, 123.68) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/caffenet-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/caffenet-12.yml new file mode 100644 index 00000000000..576c5d9a0e9 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/caffenet-12.yml @@ -0,0 +1,34 @@ +models: +- name: caffenet-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: rgb_to_bgr + + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # BGR format + - type: normalization + mean: (103.939, 116.779, 123.68) + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/densenet-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/densenet-12.yml new file mode 100644 index 00000000000..3c4668d0c84 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/densenet-12.yml @@ -0,0 +1,37 @@ +models: +- name: densenet-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # Scale to [0, 1] + - type: normalization + std: 255 + + - type: normalization + mean: (0.485, 0.456, 0.406) + std: (0.229, 0.224, 0.225) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/efficientnet-lite4-11.yml b/examples/experimental/onnx/classification/onnx_models_configs/efficientnet-lite4-11.yml new file mode 100644 index 00000000000..01a36623cc4 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/efficientnet-lite4-11.yml @@ -0,0 +1,38 @@ +models: +- name: efficientnet-lite4-11 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + inputs: + - name: images:0 + type: INPUT + shape: [1, 224, 224, 3] + layout: "NHWC" + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + - type: normalization + mean: (127.0, 127.0, 127.0) + std: (128.0, 128.0, 128.0) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/googlenet-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/googlenet-12.yml new file mode 100644 index 00000000000..e53d567b257 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/googlenet-12.yml @@ -0,0 +1,35 @@ +models: +- name: googlenet-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: rgb_to_bgr + + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # BGR format + - type: normalization + mean: (103.939, 116.779, 123.68) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/inception-v1-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/inception-v1-12.yml new file mode 100644 index 00000000000..5c77f882f39 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/inception-v1-12.yml @@ -0,0 +1,35 @@ +models: +- name: inception-v1-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: rgb_to_bgr + + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # BGR format + - type: normalization + mean: (103.939, 116.779, 123.68) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/inception-v2-9.yml b/examples/experimental/onnx/classification/onnx_models_configs/inception-v2-9.yml new file mode 100644 index 00000000000..628b4086ed9 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/inception-v2-9.yml @@ -0,0 +1,35 @@ +models: +- name: inception-v2-9 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: rgb_to_bgr + + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # BGR format + - type: normalization + mean: (103.939, 116.779, 123.68) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/mobilenetv2-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/mobilenetv2-12.yml new file mode 100644 index 00000000000..eec73fa48ac --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/mobilenetv2-12.yml @@ -0,0 +1,37 @@ +models: +- name: mobilenetv2-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # Scale to [0, 1] + - type: normalization + std: 255 + + - type: normalization + mean: (0.485, 0.456, 0.406) + std: (0.229, 0.224, 0.225) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/resnet50-v1-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/resnet50-v1-12.yml new file mode 100644 index 00000000000..052a771e72d --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/resnet50-v1-12.yml @@ -0,0 +1,37 @@ +models: +- name: resnet50-v1-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # Scale to [0, 1] + - type: normalization + std: 255 + + - type: normalization + mean: (0.485, 0.456, 0.406) + std: (0.229, 0.224, 0.225) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/resnet50-v2-7.yml b/examples/experimental/onnx/classification/onnx_models_configs/resnet50-v2-7.yml new file mode 100644 index 00000000000..0dbb7cce998 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/resnet50-v2-7.yml @@ -0,0 +1,37 @@ +models: +- name: resnet50-v2-7 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # Scale to [0, 1] + - type: normalization + std: 255 + + - type: normalization + mean: (0.485, 0.456, 0.406) + std: (0.229, 0.224, 0.225) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/shufflenet-9.yml b/examples/experimental/onnx/classification/onnx_models_configs/shufflenet-9.yml new file mode 100644 index 00000000000..1b16acdb94b --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/shufflenet-9.yml @@ -0,0 +1,37 @@ +models: +- name: shufflenet-9 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # Scale to [0, 1] + - type: normalization + std: 255 + + - type: normalization + mean: (0.485, 0.456, 0.406) + std: (0.229, 0.224, 0.225) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/shufflenet-v2-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/shufflenet-v2-12.yml new file mode 100644 index 00000000000..d12ea60e859 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/shufflenet-v2-12.yml @@ -0,0 +1,37 @@ +models: +- name: shufflenet-v2-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # Scale to [0, 1] + - type: normalization + std: 255 + + - type: normalization + mean: (0.485, 0.456, 0.406) + std: (0.229, 0.224, 0.225) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/squeezenet1.0-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/squeezenet1.0-12.yml new file mode 100644 index 00000000000..462d29969be --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/squeezenet1.0-12.yml @@ -0,0 +1,35 @@ +models: +- name: squeezenet1.0-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: rgb_to_bgr + + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # BGR format + - type: normalization + mean: (103.939, 116.779, 123.68) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/vgg16-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/vgg16-12.yml new file mode 100644 index 00000000000..1a161f07bce --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/vgg16-12.yml @@ -0,0 +1,37 @@ +models: +- name: vgg16-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # Scale to [0, 1] + - type: normalization + std: 255 + + - type: normalization + mean: (0.485, 0.456, 0.406) + std: (0.229, 0.224, 0.225) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_models_configs/zfnet512-12.yml b/examples/experimental/onnx/classification/onnx_models_configs/zfnet512-12.yml new file mode 100644 index 00000000000..40883ba50b7 --- /dev/null +++ b/examples/experimental/onnx/classification/onnx_models_configs/zfnet512-12.yml @@ -0,0 +1,35 @@ +models: +- name: zfnet512-12 + launchers: + - framework: onnx_runtime + adapter: classification + execution_providers: ['OpenVINOExecutionProvider'] + datasets: + - name: imagenet_1000_classes + data_source: /omz_data/ILSVRC2012_img_val + annotation_conversion: + converter: imagenet + annotation_file: /omz_data/val.txt + # read in RGB format + reader: pillow_imread + preprocessing: + - type: rgb_to_bgr + + - type: resize + size: 256 + aspect_ratio_scale: greater + use_pillow: true + interpolation: BILINEAR + + - type: crop + size: 224 + use_pillow: true + + # BGR format + - type: normalization + mean: (103.939, 116.779, 123.68) + + metrics: + - name: accuracy@top1 + type: accuracy + top_k: 1 diff --git a/examples/experimental/onnx/classification/onnx_ptq_classification.py b/examples/experimental/onnx/classification/onnx_ptq_classification.py index 2858845dd99..3f41e7e02e6 100644 --- a/examples/experimental/onnx/classification/onnx_ptq_classification.py +++ b/examples/experimental/onnx/classification/onnx_ptq_classification.py @@ -22,14 +22,14 @@ from nncf.experimental.post_training.compression_builder import CompressionBuilder from nncf.experimental.post_training.algorithms.quantization import PostTrainingQuantization from nncf.experimental.post_training.algorithms.quantization import PostTrainingQuantizationParameters -from nncf.experimental.onnx.datasets.imagenet_dataset import create_imagenet_torch_dataset +from nncf.experimental.onnx.datasets.imagenet_dataset import create_imagenet_torch_dataset, infer_input_shape from nncf.experimental.post_training.api.metric import Accuracy from nncf.common.utils.logger import logger as nncf_logger def run(onnx_model_path: str, output_model_path: str, dataset_path: str, batch_size: int, shuffle: bool, num_init_samples: int, - input_shape: List[int], ignored_scopes: Optional[List[str]] = None, + input_shape: Optional[List[int]] = None, ignored_scopes: Optional[List[str]] = None, evaluate: Optional[bool] = False): nncf_logger.info("Post-Training Quantization Parameters:") nncf_logger.info(" number of samples: {}".format(nncf_logger.info)) @@ -38,6 +38,11 @@ def run(onnx_model_path: str, output_model_path: str, original_model = onnx.load(onnx_model_path) nncf_logger.info("The model is loaded from {}".format(onnx_model_path)) + if input_shape is None: + nncf_logger.info( + "input_shape is None. Infer input_shape from the model.") + input_shape = infer_input_shape(original_model) + # Step 1: Initialize the data loader and metric (if it is needed). dataset = create_imagenet_torch_dataset(dataset_path, input_shape, batch_size=batch_size, shuffle=shuffle) @@ -78,18 +83,26 @@ def run(onnx_model_path: str, output_model_path: str, if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("--onnx_model_path", "-m", help="Path to ONNX model", type=str) - parser.add_argument("--output_model_path", "-o", help="Path to output quantized ONNX model", type=str) + parser.add_argument("--onnx_model_path", "-m", + help="Path to ONNX model", type=str) + parser.add_argument("--output_model_path", "-o", + help="Path to output quantized ONNX model", type=str) parser.add_argument("--data", help="Path to ImageNet validation data in the ImageFolder torchvision format " "(Please, take a look at torchvision.datasets.ImageFolder)", type=str) - parser.add_argument("--batch_size", help="Batch size for initialization", type=int, default=1) - parser.add_argument("--shuffle", help="Whether to shuffle dataset for initialization", default=True) - parser.add_argument("--input_shape", help="Model's input shape", nargs="+", type=int, default=[1, 3, 224, 224]) - parser.add_argument("--init_samples", help="Number of initialization samples", type=int, default=300) - parser.add_argument("--ignored_scopes", help="Ignored operations ot quantize", nargs="+", default=None) - parser.add_argument("--evaluate", help="Run an evaluation step for the final quantized model", action="store_true") + parser.add_argument( + "--batch_size", help="Batch size for initialization", type=int, default=1) + parser.add_argument( + "--shuffle", help="Whether to shuffle dataset for initialization", default=True) + parser.add_argument("--input_shape", help="Model's input shape. e.g. [1, 3, 224, 224]. If it's not given, it automatically infers input shape.", + nargs="+", type=int, default=None) + parser.add_argument( + "--init_samples", help="Number of initialization samples", type=int, default=300) + parser.add_argument( + "--ignored_scopes", help="Ignored operations ot quantize", nargs="+", default=None) + parser.add_argument( + "--evaluate", help="Run an evaluation step for the final quantized model", action="store_true") args = parser.parse_args() run(args.onnx_model_path, args.output_model_path, diff --git a/examples/experimental/onnx/requirements.txt b/examples/experimental/onnx/requirements.txt index abf50367e9c..a88da5b8443 100644 --- a/examples/experimental/onnx/requirements.txt +++ b/examples/experimental/onnx/requirements.txt @@ -1 +1,2 @@ -torchvision \ No newline at end of file +torchvision +openvino-dev==2022.1.0 diff --git a/examples/experimental/onnx/run_ptq.py b/examples/experimental/onnx/run_ptq.py new file mode 100644 index 00000000000..6fcbfcc8962 --- /dev/null +++ b/examples/experimental/onnx/run_ptq.py @@ -0,0 +1,122 @@ +""" + Copyright (c) 2022 Intel Corporation + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import os +from typing import List +from typing import Optional + +import numpy as np +import onnx + +from nncf.experimental.post_training.compression_builder import CompressionBuilder +from nncf.experimental.post_training.algorithms.quantization import PostTrainingQuantization +from nncf.experimental.post_training.algorithms.quantization import PostTrainingQuantizationParameters +from nncf.common.utils.logger import logger as nncf_logger + +from openvino.tools.accuracy_checker.config import ConfigReader +from openvino.tools.accuracy_checker.argparser import build_arguments_parser +from openvino.tools.accuracy_checker.dataset import Dataset +from openvino.tools.accuracy_checker.evaluators import ModelEvaluator + +import nncf.experimental.post_training.api.dataset as ptq_api_dataset + + +class OpenVINOAccuracyCheckerDataset(ptq_api_dataset.Dataset): + def __init__(self, model_evaluator: ModelEvaluator, batch_size, shuffle): + super().__init__(batch_size, shuffle) + self.model_evaluator = model_evaluator + + def __getitem__(self, item): + _, batch_annotation, batch_input, _ = self.model_evaluator.dataset[item] + filled_inputs, _, _ = self.model_evaluator._get_batch_input( + batch_annotation, batch_input) + + assert len(filled_inputs) == 1 + dummy_target = 0 + + for _, v in filled_inputs[0].items(): + return np.squeeze(v, axis=0), dummy_target + + raise RuntimeError(f"filled_inputs has no value.") + + def __len__(self): + return len(self.model_evaluator.dataset) + + +def run(onnx_model_path: str, output_model_path: str, dataset: Dataset, + ignored_scopes: Optional[List[str]] = None, evaluate: Optional[bool] = False): + + num_init_samples = len(dataset) + + nncf_logger.info("Post-Training Quantization Parameters:") + nncf_logger.info(f" number of samples: {num_init_samples}") + nncf_logger.info(f" ignored_scopes: {ignored_scopes}") + onnx.checker.check_model(onnx_model_path) + original_model = onnx.load(onnx_model_path) + nncf_logger.info(f"The model is loaded from {onnx_model_path}") + + # Step 1: Create a pipeline of compression algorithms. + builder = CompressionBuilder() + + # Step 2: Create the quantization algorithm and add to the builder. + quantization_parameters = PostTrainingQuantizationParameters( + number_samples=num_init_samples, + ignored_scopes=ignored_scopes + ) + quantization = PostTrainingQuantization(quantization_parameters) + builder.add_algorithm(quantization) + + # Step 4: Execute the pipeline. + nncf_logger.info("Post-Training Quantization has just started!") + quantized_model = builder.apply(original_model, dataset) + + # Step 5: Save the quantized model. + onnx.save(quantized_model, output_model_path) + nncf_logger.info( + "The quantized model is saved on {}".format(output_model_path)) + + onnx.checker.check_model(output_model_path) + + +if __name__ == '__main__': + parser = build_arguments_parser() + parser.add_argument("--output-model-dir", "-o", + help="Directory path to save output quantized ONNX model", type=str) + args = parser.parse_args() + config, mode = ConfigReader.merge(args) + + assert mode == "models" + for config_entry in config[mode]: + model_evaluator = ModelEvaluator.from_configs(config_entry) + assert "datasets" in config_entry + assert len(config_entry["datasets"] + ) == 1, "Config should have one dataset." + + dataset_config = config_entry["datasets"][0] + dataset = OpenVINOAccuracyCheckerDataset( + model_evaluator, batch_size=1, shuffle=True) + + assert "launchers" in config_entry + assert len(config_entry["launchers"]) == 1 + + onnx_model_path = config_entry["launchers"][0]["model"] + + fname = onnx_model_path.stem + output_model_path = os.path.join( + args.output_model_dir, fname + "-quantized.onnx") + + onnx_model_path = str(onnx_model_path) + run(onnx_model_path, + output_model_path, + dataset + ) diff --git a/examples/experimental/onnx/run_ptq_onnx_models.sh b/examples/experimental/onnx/run_ptq_onnx_models.sh new file mode 100755 index 00000000000..e2aff63d392 --- /dev/null +++ b/examples/experimental/onnx/run_ptq_onnx_models.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(dirname "$0") +CONFIGS_DIR=${SCRIPT_DIR}/classification/onnx_models_configs + +MODEL_DIR=$1 +OUTPUT_DIR=$2 +NUMBER_OF_SAMPLES=$3 + +echo "MODEL_DIR=$MODEL_DIR" +echo "OUTPUT_DIR=$OUTPUT_DIR" +echo "NUMBER_OF_SAMPLES=$NUMBER_OF_SAMPLES" + +for config in `ls $CONFIGS_DIR`; do + model_name=${config%.*} + echo $model_name + + # Post-training quantization + python $SCRIPT_DIR/run_ptq.py \ + -c $CONFIGS_DIR/$config \ + -m $MODEL_DIR/$model_name.onnx \ + -o $OUTPUT_DIR \ + -ss $NUMBER_OF_SAMPLES + + # Accuracy check for the original model + accuracy_check \ + -c $CONFIGS_DIR/$config \ + -ss $NUMBER_OF_SAMPLES \ + -m $MODEL_DIR/$model_name.onnx \ + --csv_result $OUTPUT_DIR/original_accuracy.csv + + # Accuracy check for the quantized model + accuracy_check \ + -c $CONFIGS_DIR/$config \ + -ss $NUMBER_OF_SAMPLES \ + -m $OUTPUT_DIR/$model_name-quantized.onnx \ + --csv_result $OUTPUT_DIR/quantize_accuracy.csv + + # Benchmark the original model + mkdir -p $OUTPUT_DIR/$model_name/original + + benchmark_app -m $MODEL_DIR/$model_name.onnx \ + --batch_size 1 --time 10 \ + -report_type no_counters \ + -report_folder $OUTPUT_DIR/$model_name/original + + # Benchmark the quantized model + mkdir -p $OUTPUT_DIR/$model_name/quantized + + benchmark_app -m $OUTPUT_DIR/$model_name-quantized.onnx \ + --batch_size 1 --time 10 \ + -report_type no_counters \ + -report_folder $OUTPUT_DIR/$model_name/quantized +done diff --git a/nncf/experimental/onnx/datasets/imagenet_dataset.py b/nncf/experimental/onnx/datasets/imagenet_dataset.py index dfbad128ef2..95b5df565ed 100644 --- a/nncf/experimental/onnx/datasets/imagenet_dataset.py +++ b/nncf/experimental/onnx/datasets/imagenet_dataset.py @@ -11,20 +11,28 @@ limitations under the License. """ -from typing import List +from typing import Optional, Tuple import os +import torch + from nncf.common.utils.logger import logger as nncf_logger from nncf.experimental.post_training.api.dataset import Dataset +from onnx import ModelProto +from google.protobuf.json_format import MessageToDict +from torchvision import transforms +from torchvision.datasets import ImageFolder + class ImageNetDataset(Dataset): def __init__(self, dataset, batch_size, shuffle): super().__init__(batch_size, shuffle) self.dataset = dataset - nncf_logger.info('The dataset is built with the data located on {}'.format(dataset.root)) + nncf_logger.info( + 'The dataset is built with the data located on {}'.format(dataset.root)) def __getitem__(self, item): tensor, target = self.dataset[item] @@ -35,25 +43,58 @@ def __len__(self): return len(self.dataset) -def create_imagenet_torch_dataset(dataset_dir: str, - input_shape: List[int], - mean=(0.485, 0.456, 0.406), - std=(0.229, 0.224, 0.225), - crop_ratio=0.875, - batch_size: int = 1, - shuffle: bool = True): - import torchvision - from torchvision import transforms - image_size = [input_shape[-2], input_shape[-1]] +def infer_input_shape(model: ModelProto) -> Tuple[Optional[int], int, int, int]: + assert len(model.graph.input) == 1 + + for _input in model.graph.input: + dim = _input.type.tensor_type.shape.dim + input_shape = [MessageToDict(d).get("dimValue") for d in dim] + + input_shape = [int(dim) if dim is not None else dim for dim in input_shape] + + assert len(input_shape) == 4 + + return input_shape + + +def get_transform(image_size: Tuple[int, int], + crop_ratio: float, + mean: Tuple[float, float, float], + std: Tuple[float, float, float], + channel_last: bool) -> transforms.Lambda: size = int(image_size[0] / crop_ratio) normalize = transforms.Normalize(mean=mean, std=std) - transform = transforms.Compose([ + + transform_list = [ transforms.Resize(size), transforms.CenterCrop(image_size), transforms.ToTensor(), normalize, - ]) + ] + + if channel_last: + transform_list += [transforms.Lambda( + lambda x: torch.permute(x, (1, 2, 0)))] + + return transforms.Compose(transform_list) + + +def create_imagenet_torch_dataset(dataset_dir: str, + input_shape: Optional[Tuple[int, int, int, int]], + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + crop_ratio=0.875, + batch_size: int = 1, + shuffle: bool = True): + channel_last = input_shape[1] > input_shape[3] and input_shape[2] > input_shape[3] + + if channel_last: + image_size = [input_shape[-3], input_shape[-2]] + else: + image_size = [input_shape[-2], input_shape[-1]] + + transform = get_transform(image_size, crop_ratio, mean, std, channel_last) # The best practise is to use validation part of dataset for calibration (aligning with POT) - initialization_dataset = torchvision.datasets.ImageFolder(os.path.join(dataset_dir), transform) + initialization_dataset = ImageFolder(os.path.join(dataset_dir), transform) return ImageNetDataset(initialization_dataset, batch_size, shuffle) diff --git a/nncf/experimental/onnx/engine.py b/nncf/experimental/onnx/engine.py index 9a943d0bb20..dfb94d4baee 100644 --- a/nncf/experimental/onnx/engine.py +++ b/nncf/experimental/onnx/engine.py @@ -34,8 +34,11 @@ def __init__(self, **rt_session_options): self._inputs_transforms = lambda input_data: input_data.astype(np.float32) self.sess = None self.rt_session_options = rt_session_options - if 'providers' not in self.rt_session_options: - self.rt_session_options['providers'] = ['OpenVINOExecutionProvider'] + + # TODO: Do not force it to use CPUExecutionProvider + # OpenVINOExecutionProvider raises the following error. + # onnxruntime.capi.onnxruntime_pybind11_state.Fail: [ONNXRuntimeError] : 1 : FAIL : This is an invalid model. Error: Duplicate definition of name (data). + self.rt_session_options['providers'] = ['CPUExecutionProvider'] def get_sampler(self) -> Sampler: # TODO (Nikita Malinin): Replace range calling with the max length variable diff --git a/nncf/experimental/onnx/model_normalizer.py b/nncf/experimental/onnx/model_normalizer.py index 1fd9510f2bd..206e1b1f88c 100644 --- a/nncf/experimental/onnx/model_normalizer.py +++ b/nncf/experimental/onnx/model_normalizer.py @@ -83,6 +83,8 @@ def add_const_value_infos_to_graph(graph: onnx.GraphProto): model.ir_version = 7 # Due to the 'Shufflenet-v1 modified_model = version_converter.convert_version(model, 13) + # ONNX shape inference + # https://github.com/onnx/onnx/blob/main/docs/proposals/SymbolicShapeInfProposal.md modified_model = onnx.shape_inference.infer_shapes(modified_model) add_input_from_initializer(modified_model) diff --git a/tests/onnx/sanity_tests/test_classification_sanity_sample.py b/tests/onnx/sanity_tests/test_classification_sanity_sample.py index f04aa1e1e4e..c35963cf6e1 100644 --- a/tests/onnx/sanity_tests/test_classification_sanity_sample.py +++ b/tests/onnx/sanity_tests/test_classification_sanity_sample.py @@ -88,7 +88,8 @@ def test_sanity_quantize_sample(tmp_path, model_name, model, input_shape): batch_size=1, shuffle=True, num_init_samples=1, input_shape=input_shape, ignored_scopes=None) - sess = rt.InferenceSession(onnx_output_model_path, providers=['OpenVINOExecutionProvider']) + sess = rt.InferenceSession(onnx_output_model_path, providers=[ + 'OpenVINOExecutionProvider']) _input = np.random.random(input_shape) input_name = sess.get_inputs()[0].name _ = sess.run([], {input_name: _input.astype(np.float32)})