From 659daf610f1cb31848c88f42c5c27d8bd5f0b6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ko=C5=BCykowski?= Date: Wed, 6 Oct 2021 13:53:47 +0200 Subject: [PATCH] [ONNX] MatMulInteger (#7825) --- .../onnx/frontend/src/op/matmul_integer.cpp | 55 ++++ .../onnx/frontend/src/op/matmul_integer.hpp | 24 ++ .../frontend/onnx/frontend/src/ops_bridge.cpp | 2 + .../test/models/onnx/matmul_integer.prototxt | 91 +++++ .../onnx/matmul_integer_2d_x_3d.prototxt | 83 +++++ .../models/onnx/matmul_integer_3d.prototxt | 106 ++++++ .../onnx/matmul_integer_3d_x_2d.prototxt | 97 ++++++ .../models/onnx/matmul_integer_4d.prototxt | 109 ++++++ .../matmul_integer_4d_no_zero_point.prototxt | 81 +++++ .../models/onnx/matmul_integer_int8.prototxt | 91 +++++ .../matmul_integer_matrix_zero_point.prototxt | 127 +++++++ .../matmul_integer_no_zero_point.prototxt | 63 ++++ ...mul_integer_vectorized_zero_point.prototxt | 91 +++++ ngraph/test/onnx/onnx_import_quant.in.cpp | 310 ++++++++++++++++++ ngraph/test/runtime/ie/unit_test.manifest | 7 - .../runtime/interpreter/unit_test.manifest | 9 +- .../python/tests/test_onnx/test_backend.py | 1 - 17 files changed, 1331 insertions(+), 16 deletions(-) create mode 100644 ngraph/frontend/onnx/frontend/src/op/matmul_integer.cpp create mode 100644 ngraph/frontend/onnx/frontend/src/op/matmul_integer.hpp create mode 100644 ngraph/test/models/onnx/matmul_integer.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_2d_x_3d.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_3d.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_3d_x_2d.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_4d.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_4d_no_zero_point.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_int8.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_matrix_zero_point.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_no_zero_point.prototxt create mode 100644 ngraph/test/models/onnx/matmul_integer_vectorized_zero_point.prototxt diff --git a/ngraph/frontend/onnx/frontend/src/op/matmul_integer.cpp b/ngraph/frontend/onnx/frontend/src/op/matmul_integer.cpp new file mode 100644 index 00000000000000..0952b3bd2406fd --- /dev/null +++ b/ngraph/frontend/onnx/frontend/src/op/matmul_integer.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "op/matmul_integer.hpp" + +#include +#include +#include + +#include "default_opset.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_1 { +OutputVector matmul_integer(const Node& node) { + const OutputVector& inputs = node.get_ng_inputs(); + + const auto& A = inputs.at(0); + const auto& B = inputs.at(1); + const auto& A_zero_point = + (inputs.size() > 2) ? inputs.at(2) : ngraph::op::Constant::create(ngraph::element::i32, {1}, {0}); + const auto& B_zero_point = + (inputs.size() > 3) ? inputs.at(3) : ngraph::op::Constant::create(ngraph::element::i32, {1}, {0}); + + const auto& converted_A = std::make_shared(A, element::i32); + const auto& converted_B = std::make_shared(B, element::i32); + + const auto& converted_A_zero_point = std::make_shared(A_zero_point, element::i32); + const auto& converted_B_zero_point = std::make_shared(B_zero_point, element::i32); + + const auto& A_zero_point_rank = A_zero_point.get_partial_shape().rank(); + + Output shifted_A; + if (A_zero_point_rank.is_static() && A_zero_point_rank.get_length() == 1) { + const auto& one_node = ngraph::op::Constant::create(ngraph::element::i32, {1}, {1}); + const auto& reshaped_A_zero_point = + std::make_shared(converted_A_zero_point, one_node); + + shifted_A = std::make_shared(converted_A, reshaped_A_zero_point); + } else { + shifted_A = std::make_shared(converted_A, converted_A_zero_point); + } + + const auto& shifted_B = std::make_shared(converted_B, converted_B_zero_point); + + const auto& result = std::make_shared(shifted_A, shifted_B); + + return {result}; +} +} // namespace set_1 +} // namespace op +} // namespace onnx_import +} // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/matmul_integer.hpp b/ngraph/frontend/onnx/frontend/src/op/matmul_integer.hpp new file mode 100644 index 00000000000000..e1db5c2416941f --- /dev/null +++ b/ngraph/frontend/onnx/frontend/src/op/matmul_integer.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "ngraph/node.hpp" +#include "onnx_import/core/node.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_1 { +/// \brief Performs ONNX MatMulInteger operation. +/// +/// \param node The ONNX node object representing this operation. +/// +/// \return The vector containing Ngraph nodes producing output of ONNX quantizied +/// matrix multiplication integer operation. +OutputVector matmul_integer(const Node& node); +} // namespace set_1 +} // namespace op +} // namespace onnx_import +} // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp b/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp index 20484c78953ffd..b1e3734a2f35f9 100644 --- a/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp +++ b/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp @@ -81,6 +81,7 @@ #include "op/lrn.hpp" #include "op/lstm.hpp" #include "op/matmul.hpp" +#include "op/matmul_integer.hpp" #include "op/max.hpp" #include "op/max_pool.hpp" #include "op/mean.hpp" @@ -352,6 +353,7 @@ OperatorsBridge::OperatorsBridge() { REGISTER_OPERATOR("LpNormalization", 1, lp_norm); REGISTER_OPERATOR("LRN", 1, lrn); REGISTER_OPERATOR("LSTM", 1, lstm); + REGISTER_OPERATOR("MatMulInteger", 1, matmul_integer); REGISTER_OPERATOR("MatMul", 1, matmul); REGISTER_OPERATOR("MaxPool", 1, max_pool); REGISTER_OPERATOR("Max", 1, max); diff --git a/ngraph/test/models/onnx/matmul_integer.prototxt b/ngraph/test/models/onnx/matmul_integer.prototxt new file mode 100644 index 00000000000000..813f4920ce7db0 --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer.prototxt @@ -0,0 +1,91 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + input: "b_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "b_zero_point" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 2 + } + } + } + } + } +} +opset_import { + domain: "" + version: 13 +} diff --git a/ngraph/test/models/onnx/matmul_integer_2d_x_3d.prototxt b/ngraph/test/models/onnx/matmul_integer_2d_x_3d.prototxt new file mode 100644 index 00000000000000..dcac5092c5a206 --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_2d_x_3d.prototxt @@ -0,0 +1,83 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_3d.prototxt b/ngraph/test/models/onnx/matmul_integer_3d.prototxt new file mode 100644 index 00000000000000..e640c7714d4a86 --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_3d.prototxt @@ -0,0 +1,106 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + input: "b_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "b_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 1 + } + dim { + dim_value: 4 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_3d_x_2d.prototxt b/ngraph/test/models/onnx/matmul_integer_3d_x_2d.prototxt new file mode 100644 index 00000000000000..e359be5d04d20e --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_3d_x_2d.prototxt @@ -0,0 +1,97 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + input: "b_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "b_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_4d.prototxt b/ngraph/test/models/onnx/matmul_integer_4d.prototxt new file mode 100644 index 00000000000000..a1c29d86d62a21 --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_4d.prototxt @@ -0,0 +1,109 @@ +ir_version: 5 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + input: "b_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "test" + input { + name: "A" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "b_zero_point" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_4d_no_zero_point.prototxt b/ngraph/test/models/onnx/matmul_integer_4d_no_zero_point.prototxt new file mode 100644 index 00000000000000..6f14518837ba1f --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_4d_no_zero_point.prototxt @@ -0,0 +1,81 @@ +ir_version: 5 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + output: "Y" + op_type: "MatMulInteger" + } + name: "test" + input { + name: "A" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_int8.prototxt b/ngraph/test/models/onnx/matmul_integer_int8.prototxt new file mode 100644 index 00000000000000..e28705d03b8790 --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_int8.prototxt @@ -0,0 +1,91 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + input: "b_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "b_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_matrix_zero_point.prototxt b/ngraph/test/models/onnx/matmul_integer_matrix_zero_point.prototxt new file mode 100644 index 00000000000000..31bfa2e775da4d --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_matrix_zero_point.prototxt @@ -0,0 +1,127 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + input: "b_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "b_zero_point" + type { + tensor_type { + elem_type: 3 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 1 + } + dim { + dim_value: 3 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_no_zero_point.prototxt b/ngraph/test/models/onnx/matmul_integer_no_zero_point.prototxt new file mode 100644 index 00000000000000..aefb304823cd3a --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_no_zero_point.prototxt @@ -0,0 +1,63 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 5 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 5 + } + dim { + dim_value: 3 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/models/onnx/matmul_integer_vectorized_zero_point.prototxt b/ngraph/test/models/onnx/matmul_integer_vectorized_zero_point.prototxt new file mode 100644 index 00000000000000..12974e6b4caf9f --- /dev/null +++ b/ngraph/test/models/onnx/matmul_integer_vectorized_zero_point.prototxt @@ -0,0 +1,91 @@ +ir_version: 7 +producer_name: "nGraph ONNX Importer" +graph { + node { + input: "A" + input: "B" + input: "a_zero_point" + input: "b_zero_point" + output: "Y" + op_type: "MatMulInteger" + } + name: "MatMulInt" + input { + name: "A" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 5 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 5 + } + dim { + dim_value: 3 + } + } + } + } + } + input { + name: "a_zero_point" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "b_zero_point" + type { + tensor_type { + elem_type: 2 + shape { + dim { + dim_value: 3 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + domain: "" + version: 10 +} diff --git a/ngraph/test/onnx/onnx_import_quant.in.cpp b/ngraph/test/onnx/onnx_import_quant.in.cpp index 551952ee090353..049a1712442205 100644 --- a/ngraph/test/onnx/onnx_import_quant.in.cpp +++ b/ngraph/test/onnx/onnx_import_quant.in.cpp @@ -402,6 +402,316 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_quant_conv_linear_onnx_example) { test_case.run(); } +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_2d_simple_zero_point) { + auto function = onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{11, 7, 3, + 10, 6, 2, + 9, 5, 1, + 8, 4, 0}); // A + test_case.add_input(std::vector{1, 4, + 2, 5, + 3, 6}); // B + test_case.add_input(std::vector{12}); // a_zero_point + test_case.add_input(std::vector{0}); // b_zero_point + + test_case.add_expected_output({4, 2}, std::vector{-38, -83, + -44, -98, + -50, -113, + -56, -128}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_int8) { + auto function = + onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_int8.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{-3, 7, 5, -6, + 4, -5, 8, 7}); // A + test_case.add_input(std::vector{ 5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); // B + test_case.add_input(std::vector{5}); // a_zero_point + test_case.add_input(std::vector{5}); // b_zero_point + + test_case.add_expected_output({2, 4}, std::vector{-55, 16, 89, -44, + 122, 154, 68, -39}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_vectorized_zero_point) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_vectorized_zero_point.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{11, 22, 33, 44, 55, + 22, 33, 44, 55, 66, + 33, 44, 55, 66, 77, + 44, 55, 66, 77, 88}); // A + test_case.add_input(std::vector{ 13, 1, 3, + 21, 49, 31, + 9, 0, 2, + 107, 7, 94, + 1, 63, 127}); // B + test_case.add_input(std::vector{33, 44, 55, 66}); // a_zero_point + test_case.add_input(std::vector{10, 20, 30}); // b_zero_point + + test_case.add_expected_output({4, 3}, std::vector{682, 902, 3421, + 682, 902, 3421, + 682, 902, 3421, + 682, 902, 3421}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_no_zero_point) { + auto function = + onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_no_zero_point.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{11, 22, 33, 44, 55, + 22, 33, 44, 55, 66, + 33, 44, 55, 66, 77, + 44, 55, 66, 77, 88}); // A + test_case.add_input(std::vector{ 13, 1, 3, + 21, 49, 31, + 9, 0, 2, + 107, 7, 94, + 1, 63, 127}); // B + + test_case.add_expected_output({4, 3}, std::vector{ 5665, 4862, 11902, + 7326, 6182, 14729, + 8987, 7502, 17556, + 10648, 8822, 20383}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_2d_x_3d) { + auto function = + onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_2d_x_3d.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{7, -3, 1, 2, + 0, 2, -4, 6}); // A + test_case.add_input(std::vector{1, -13, 10, + 2, -16, 14, + 3, -19, 18, + 4, -22, 22, + + -1, 13, -10, + -2, 16, -14, + -3, 19, -18, + -4, 22, -22}); // B + test_case.add_input(std::vector{-4}); // a_zero_point + + test_case.add_expected_output({2, 2, 3}, std::vector{52, -386, 346, + 56, -368, 344, + + -52, 386, -346, + -56, 368, -344}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_3d_x_2d) { + auto function = + onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_3d_x_2d.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{-13, 11, -1, -2, + 4, -2, 3, 10, + + 8, -2, 4, 5, + -4, -3, 1, 2}); // A + test_case.add_input(std::vector{ 1, -3, 5, + 7, -2, -10, + -13, 9, 7, + 11, 3, -3}); // B + test_case.add_input(std::vector{4}); // a_zero_point + test_case.add_input(std::vector{-3}); // a_zero_point + + test_case.add_expected_output({2, 2, 3}, std::vector{-32, -89, -235, + 34, 18, 32, + + -30, 0, 74, + -100, -55, -45}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_3d) { + auto function = onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_3d.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{125, 135, 145, 155, + 130, 140, 150, 160, + + 125, 135, 145, 155, + 130, 140, 150, 160}); // A + test_case.add_input(std::vector{-10, -5, 0, 5, + -5, 0, 5, 10, + -5, -4, -3, -2, + -1, 0, 1, 2, + + 10, 5, 0, -5, + 5, 0, -5, -10, + 5, 4, 3, 2, + 1, 0, -1, -2}); // B + test_case.add_input(std::vector{150}); // a_zero_point + test_case.add_input(std::vector{5, 10, 15, 20, + -5, -10, -15, -20}); // b_zero_point + + test_case.add_expected_output({2, 2, 4}, std::vector{545, 545, 545, 545, + 340, 300, 260, 220, + + -545, -545, -545, -545, + -340, -300, -260, -220}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_4d) { + auto function = onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_4d.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + + 12, 13, 14, + 15, 16, 17, + 18, 19, 20, + 21, 22, 23}); // A + test_case.add_input(std::vector{0, 1, 2, + 3, 4, 5, + 6, 7, 8, + 9, 10, 11, + + 12, 13, 14, + 15, 16, 17, + 18, 19, 20, + 21, 22, 23}); // B + test_case.add_input(std::vector{0}); // a_zero_point + test_case.add_input(std::vector{0}); // b_zero_point + + test_case.add_expected_output({1, 2, 3, 3}, std::vector {42, 48, 54, + 114, 136, 158, + 186, 224, 262, + + 906, 960, 1014, + 1170, 1240, 1310, + 1434, 1520, 1606}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_4d_zero_point) { + auto function = onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_4d.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23}); // A + test_case.add_input(std::vector{0, 1, 2, + 3, 4, 5, + 6, 7, 8, + 9, 10, 11, + + 12, 13, 14, + 15, 16, 17, + 18, 19, 20, + 21, 22, 23}); // B + test_case.add_input(std::vector{1}); // a_zero_point + test_case.add_input(std::vector{1}); // b_zero_point + + test_case.add_expected_output({1, 2, 3, 3}, std::vector{22, 24, 26, + 78, 96, 114, + 134, 168, 202, + + 790, 840, 890, + 1038, 1104, 1170, + 1286, 1368, 1450}); // Y + // clang-format on + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_matmul_integer_matrix_zero_point) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(SERIALIZED_ZOO, "onnx/matmul_integer_matrix_zero_point.onnx")); + + auto test_case = test::TestCase(function); + + // don't change style for better readibility + // clang-format off + test_case.add_input(std::vector{0, 1, 2, 3, + 4, 5, 6, 7, + + 8, 9, 10, 11, + 12, 13, 14, 15}); // A + test_case.add_input(std::vector{0, 1, 2, + 3, 4, 5, + 6, 7, 8, + 9, 10, 11, + + 12, 13, 14, + 15, 16, 17, + 18, 19, 20, + 21, 22, 23}); // B + test_case.add_input(std::vector{1, + 2, + + 3, + 4}); // a_zero_point + test_case.add_input(std::vector{1, 2, 3, + + 4, 5, 6}); // b_zero_point + + test_case.add_expected_output({1, 2, 2, 3}, std::vector{22, 22, 22, + 64, 64, 64, + + 340, 340, 340, + 490, 490, 490}); // Y + // clang-format on + test_case.run(); +} + NGRAPH_TEST(${BACKEND_NAME}, onnx_model_qlinear_matmul_3d) { auto function = onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/qlinear_matmul_3d.onnx")); diff --git a/ngraph/test/runtime/ie/unit_test.manifest b/ngraph/test/runtime/ie/unit_test.manifest index 06313eb370c30e..b4feff3b3074a3 100644 --- a/ngraph/test/runtime/ie/unit_test.manifest +++ b/ngraph/test/runtime/ie/unit_test.manifest @@ -31,13 +31,6 @@ onnx_model_conv_integer_no_zero_point onnx_model_conv_integer_pads # Unsupported operator detected in the graph: QuantizedDot -onnx_model_matmul_integer -onnx_model_matmul_integer_zero_point_zero -onnx_model_matmul_integer_no_zero_point -onnx_model_matmul_integer_scalar -onnx_model_matmul_integer_4d -onnx_model_matmul_integer_4d_zero_point -onnx_model_matmul_integer_4d_no_zero_point onnx_model_qlinear_matmul onnx_model_qlinear_matmul_3d diff --git a/ngraph/test/runtime/interpreter/unit_test.manifest b/ngraph/test/runtime/interpreter/unit_test.manifest index 637269119df812..4b04419aae449f 100644 --- a/ngraph/test/runtime/interpreter/unit_test.manifest +++ b/ngraph/test/runtime/interpreter/unit_test.manifest @@ -14,13 +14,6 @@ INTERPRETER.onnx_resize11_sizes_nearest_asymmetric_floor # nGraph does not support the following ONNX operations INTERPRETER.onnx_model_qlinear_matmul INTERPRETER.onnx_model_qlinear_matmul_3d -INTERPRETER.onnx_model_matmul_integer -INTERPRETER.onnx_model_matmul_integer_zero_point_zero -INTERPRETER.onnx_model_matmul_integer_no_zero_point -INTERPRETER.onnx_model_matmul_integer_scalar -INTERPRETER.onnx_model_matmul_integer_4d -INTERPRETER.onnx_model_matmul_integer_4d_zero_point -INTERPRETER.onnx_model_matmul_integer_4d_no_zero_point # Disabled tests for disabled reference implementations INTERPRETER.onnx_dyn_shapes_expand_uint16_dyn_shape @@ -139,4 +132,4 @@ INTERPRETER.zero_sized_negative # No support yet for RandomUniform INTERPRETER.onnx_model_random_uniform -INTERPRETER.onnx_model_random_uniform_like \ No newline at end of file +INTERPRETER.onnx_model_random_uniform_like diff --git a/runtime/bindings/python/tests/test_onnx/test_backend.py b/runtime/bindings/python/tests/test_onnx/test_backend.py index e20d0cc099aaad..6fd59dd924c564 100644 --- a/runtime/bindings/python/tests/test_onnx/test_backend.py +++ b/runtime/bindings/python/tests/test_onnx/test_backend.py @@ -333,7 +333,6 @@ def expect_fail(test_case_path, xfail): # type: (str) -> None ), ( xfail_issue_38722, - "OnnxBackendNodeModelTest.test_matmulinteger_cpu", "OnnxBackendNodeModelTest.test_qlinearmatmul_2D_cpu", "OnnxBackendNodeModelTest.test_qlinearmatmul_3D_cpu", ),