Comparing of results for lpt - tests

inference-engine/tests/functional/inference_engine/lp_transformations/avg_pool_transformation.cpp

@@ -79,6 +79,11 @@ TEST_P(AvgPoolTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(AvgPoolTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16

inference-engine/tests/functional/inference_engine/lp_transformations/concat_transformation.cpp

@@ -126,6 +126,11 @@ TEST_P(ConcatTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(ConcatTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16

inference-engine/tests/functional/inference_engine/lp_transformations/concat_with_intermediate_transformation.cpp

@@ -137,6 +137,11 @@ TEST_P(ConcatWithIntermediateTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(ConcatWithIntermediateTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16

inference-engine/tests/functional/inference_engine/lp_transformations/concat_with_neighbors_transformation.cpp

@@ -139,12 +139,19 @@ TEST_P(ConcatWithNeighborsTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(ConcatWithNeighborsTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16
 };
 
-const std::vector<bool> updatePrecisions = { true, false };
+const std::vector<bool> updatePrecisions = {
+//        true,
+        false };
 
 const std::vector<ConcatTransformationTestValues> testValues = {
     // U8: concat

inference-engine/tests/functional/inference_engine/lp_transformations/convolution_transformation.cpp

@@ -84,6 +84,11 @@ TEST_P(ConvolutionTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(ConvolutionTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16

inference-engine/tests/functional/inference_engine/lp_transformations/fake_quantize_precision_selection_transformation.cpp

@@ -133,12 +133,19 @@ TEST_P(FakeQuantizePrecisionSelectionTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(FakeQuantizePrecisionSelectionTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16
 };
 
-const std::vector<bool> updatePrecisions = { true, false };
+const std::vector<bool> updatePrecisions = {
+//        true,
+        false };
 
 const std::vector<FakeQuantizePrecisionSelectionTransformationTestValues> fakeQuantizeTransformationTestValues = {
     {

inference-engine/tests/functional/inference_engine/lp_transformations/fake_quantize_transformation.cpp

@@ -112,6 +112,11 @@ TEST_P(FakeQuantizeTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(FakeQuantizeTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16

inference-engine/tests/functional/inference_engine/lp_transformations/fuse_fake_quantize_with_multi_inputs_transformation.cpp

@@ -95,6 +95,11 @@ TEST_P(FuseFakeQuantizeWithMultiInputsTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(FuseFakeQuantizeWithMultiInputsTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<FuseFakeQuantizeTransformationTestValues> testValues = {
     // Multiply
     {

inference-engine/tests/functional/inference_engine/lp_transformations/fuse_subtract_to_fake_quantize_transformation.cpp

@@ -83,6 +83,11 @@ TEST_P(FuseSubtractToFakeQuantizeTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(FuseSubtractToFakeQuantizeTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<FuseSubtractToFakeQuantizeTransformationTestValues> testValues = {
     {
         Shape{1, 3, 16, 16},

inference-engine/tests/functional/inference_engine/lp_transformations/group_convolution_transformation.cpp

@@ -91,12 +91,19 @@ TEST_P(GroupConvolutionTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(GroupConvolutionTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16
 };
 
-const std::vector<bool> updatePrecisions = { true, false };
+const std::vector<bool> updatePrecisions = {
+//        true,
+        false };
 
 const std::vector<GroupConvolutionTransformationTestParams> testParams = {
     // group convolution, tensor quantization, with zero point

inference-engine/tests/functional/inference_engine/lp_transformations/layer_transformation.cpp

@@ -5,7 +5,9 @@
 #include "layer_transformation.hpp"
 
 #include <ngraph/opsets/opset1.hpp>
+#include <details/ie_exception.hpp>
 #include "simple_low_precision_transformer.hpp"
+#include <ngraph_functions/pass/convert_prc.hpp>
 
 using namespace testing;
 using namespace ngraph::pass;
@@ -76,3 +78,111 @@ std::string LayerTransformation::getTestCaseNameByParams(
     result << type << "_" << shape << "_" << toString(params);
     return result.str();
 }
+
+template<class T>
+void LayerTransformation::compareTypedVectors(
+        const std::vector<std::vector<T>>& v1,
+        const std::vector<std::vector<T>>& v2) {
+    ASSERT_EQ(v1.size(), v2.size());
+    auto size = v1.size();
+    for (std::size_t idx = 0; idx < size; ++idx) {
+        const auto& expected = v1[idx];
+        const auto& actual = v2[idx];
+        if ( !std::is_same<T, int>::value &&
+        !std::is_same<T, float>::value) {
+            THROW_IE_EXCEPTION << "Precision not supported";
+        }
+        compareValues<T>(v1[idx].data(), v2[idx].data(), v1.size());
+    }
+}
+
+template<class T>
+void LayerTransformation::compareBytes(
+        const std::vector<std::vector<std::uint8_t>>& expectedVector,
+        const std::vector<std::vector<std::uint8_t>>& actualVector) {
+    for (std::size_t idx = 0; idx < expectedVector.size(); ++idx) {
+        const auto& expected = expectedVector[idx];
+        const auto& actual = actualVector[idx];
+        ASSERT_EQ(expectedVector.size(), actualVector.size());
+        const unsigned char *expectedBuffer = expected.data();
+        const unsigned char *actualBuffer = actual.data();
+        auto size = actual.size();
+        compareValues<T>(expectedBuffer, actualBuffer, size);
+    }
+}
+
+template<class T>
+void LayerTransformation::compareValues(const T *expected, const T *actual, std::size_t size, T thr) {
+    std::cout << std::endl;
+    for (std::size_t i = 0; i < size; ++i) {
+        const auto &ref = expected[i];
+        const auto &res = actual[i];
+        const auto absoluteDifference = std::abs(res - ref);
+        if (absoluteDifference <= thr) {
+            continue;
+        }
+
+        const auto max = std::max(std::abs(res), std::abs(ref));
+        ASSERT_TRUE(max != 0 && ((absoluteDifference / max) <= thr))
+                                    << "Relative comparison of values expected: " << ref << " and actual: " << res
+                                    << " at index " << i << " with t " << thr
+                                    << " failed";
+    }
+}
+
+template<class T>
+void LayerTransformation::compareValues(const void *expected, const void *actual, std::size_t size) {
+    if (std::is_same<T, float>::value) {
+        compareValues(
+                reinterpret_cast<const float *>(expected), reinterpret_cast<const float *>(actual),
+                size, threshold);
+    } else if (std::is_same<T, int>::value) {
+        compareValues(
+                reinterpret_cast<const std::int32_t *>(expected),
+                reinterpret_cast<const std::int32_t *>(actual), size, 0);
+    } else {
+        FAIL() << "Comparator for given precision isn't supported";
+    }
+}
+std::vector<std::vector<std::uint8_t>> LayerTransformation::CalculateRefs(
+        std::shared_ptr<ngraph::Function> _function,
+        std::vector<std::vector<std::uint8_t>> _inputs) {
+    // nGraph interpreter does not support f16
+    // IE converts f16 to f32
+    ngraph::pass::ConvertPrecision<ngraph::element::Type_t::f16, ngraph::element::Type_t::f32>().run_on_function(_function);
+    _function->validate_nodes_and_infer_types();
+    return ngraph::helpers::interpreterFunction(_function, _inputs, ::ngraph::element::Type_t::undefined);
+}
+
+bool LayerTransformation::compareResults(std::shared_ptr<ngraph::Function> f1, std::shared_ptr<ngraph::Function> f2) {
+    auto pr = actualFunction->get_parameters()[0]->get_element_type();
+    auto sh = actualFunction->get_parameters()[0]->get_partial_shape().to_shape();
+    size_t byteVectorSize = shape_size(sh);
+
+    const auto &parameters = actualFunction->get_parameters();
+    for (const auto &parameter : parameters) {
+        const auto &parameterIndex = actualFunction->get_parameter_index(parameter);
+        const auto &parameterShape = parameter->get_shape();
+        const auto &parameterType = parameter->get_element_type();
+        const auto &parameterSize = shape_size(parameterShape) * parameterType.size();
+        std::vector<std::uint8_t> inputVector(parameterSize);
+        byteInputData.push_back(inputVector);
+    }
+
+    auto type = actualFunction->get_parameters()[0]->get_element_type();
+    if (type == ::ngraph::element::f32) {
+        FuncTestUtils::fillByteVectorAsTyped<float>(byteInputData);
+    } else if (type == ::ngraph::element::i32) {
+        FuncTestUtils::fillByteVectorAsTyped<int>(byteInputData);
+    }
+    auto res1 = CalculateRefs(f1, byteInputData);
+    auto res2 = CalculateRefs(f2, byteInputData);
+
+    auto vec1 = FuncTestUtils::getTypedVector<float>(res1);
+    auto vec2 = FuncTestUtils::getTypedVector<float>(res2);
+
+    // get real typed vectors
+    compareTypedVectors<float>(vec1, vec2);
+//    CompareBytes(res1, res2, InferenceEngine::Precision::FP32);
+    return true;
+}

inference-engine/tests/functional/inference_engine/lp_transformations/layer_transformation.hpp

@@ -8,7 +8,7 @@
 #include "transformations/low_precision/layer_transformation.hpp"
 #include "transformations/low_precision/transformation_context.hpp"
 #include "transformations/low_precision/transformer.hpp"
-
+#include <functional_test_utils/blob_utils.hpp>
 typedef std::tuple<
     ngraph::element::Type,
     ngraph::Shape,
@@ -33,7 +33,21 @@ class LayerTransformation : public CommonTestUtils::TestsCommon {
     void transform(
         std::shared_ptr<ngraph::Function> function,
         std::map<std::string, ngraph::pass::low_precision::LayerTransformationPtr>& transformations);
+    template<class T>
+    void compareTypedVectors(const std::vector<std::vector<T>>& v1, const std::vector<std::vector<T>>& v2);
+    template<class T>
+    void compareBytes(const std::vector<std::vector<std::uint8_t>>& expectedVector, const std::vector<std::vector<std::uint8_t>>& actualVector);
+    template<class T>
+    void compareValues(const T *expected, const T *actual, std::size_t size, T thr);
+    template<class T>
+    void compareValues(const void *expected, const void *actual, std::size_t size);
+    virtual std::vector<std::vector<std::uint8_t>> CalculateRefs(std::shared_ptr<ngraph::Function> _function, std::vector<std::vector<std::uint8_t>> _inputs);
+    bool compareResults(std::shared_ptr<ngraph::Function> f1, std::shared_ptr<ngraph::Function> f2);
 
     std::shared_ptr<ngraph::Function> actualFunction;
     std::shared_ptr<ngraph::Function> referenceFunction;
+    std::vector<std::vector<std::uint8_t>> byteInputData;
+    std::vector<std::vector<std::uint8_t>> actualByteOutput;
+    std::vector<std::vector<std::uint8_t>> expectedByteOutput;
+    float threshold = 1e-2f;
 };

inference-engine/tests/functional/inference_engine/lp_transformations/mat_mul_transformation.cpp

@@ -127,6 +127,11 @@ TEST_P(MatMulTransformation, CompareFunctions) {
     ASSERT_TRUE(res.first) << res.second;
 }
 
+TEST_P(MatMulTransformation, CompareOutputs) {
+    auto res = compareResults(actualFunction, referenceFunction);
+    ASSERT_TRUE(res);
+}
+
 const std::vector<ngraph::element::Type> precisions = {
     ngraph::element::f32,
     // ngraph::element::f16

inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp

@@ -75,4 +75,4 @@ class ngraph::pass::InjectionPass : public ngraph::pass::FunctionPass {
 
 private:
     injection_callback m_callback;
-};
+};