diff --git a/docs/template_plugin/tests/functional/op_reference/not_equal.cpp b/docs/template_plugin/tests/functional/op_reference/not_equal.cpp
new file mode 100644
index 00000000000000..e0292ecc67a4be
--- /dev/null
+++ b/docs/template_plugin/tests/functional/op_reference/not_equal.cpp
@@ -0,0 +1,123 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <gtest/gtest.h>
+
+#include <ie_core.hpp>
+#include <ie_ngraph_utils.hpp>
+#include <ngraph/ngraph.hpp>
+#include <shared_test_classes/base/layer_test_utils.hpp>
+
+#include "comparison.hpp"
+
+using namespace ngraph;
+using namespace InferenceEngine;
+using ComparisonTypes = ngraph::helpers::ComparisonTypes;
+
+
+namespace reference_tests {
+namespace ComparisonOpsRefTestDefinitions {
+namespace {
+
+template <element::Type_t IN_ET>
+std::vector<RefComparisonParams> generateComparisonParams(const element::Type& type) {
+    using T = typename element_type_traits<IN_ET>::value_type;
+    std::vector<RefComparisonParams> compParams {
+        // 1D // 2D // 3D // 4D
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{2, 2}, type, std::vector<T> {1, 0, 10, 255}})
+            .input2({{2, 2}, type, std::vector<T> {1, 0, 10, 255}})
+            .expected({{2, 2}, element::boolean, std::vector<char> {0, 0, 0, 0}}),
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{2, 3}, type, std::vector<T> {0, 15, 45, 10, 5, 10}})
+            .input2({{2, 3}, type, std::vector<T> {1, 15, 5, 10, 50, 10}})
+            .expected({{2, 3}, element::boolean, std::vector<char> {1, 0, 1, 0, 1, 0}}),
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{1}, type, std::vector<T> {20}})
+            .input2({{1}, type, std::vector<T> {10}})
+            .expected({{1}, element::boolean, std::vector<char> {1}}),
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{2, 4}, type, std::vector<T> {0, 12, 23, 0, 1, 5, 12, 8}})
+            .input2({{2, 4}, type, std::vector<T> {0, 12, 23, 0, 10, 5, 11, 8}})
+            .expected({{2, 4}, element::boolean, std::vector<char> {0, 0, 0, 0, 1, 0, 1, 0}}),
+        Builder {}
+                .compType(ComparisonTypes::NOT_EQUAL)
+                .input1({{3, 1, 2}, type, std::vector<T> {2, 7, 4, 7, 3, 7}})
+                .input2({{1, 2, 1}, type, std::vector<T> {7, 7}})
+                .expected({{3, 2, 2}, element::boolean, std::vector<char> {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0}}),
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{2, 1, 2, 1}, type, std::vector<T> {1, 2, 1, 4}})
+            .input2({{1, 2, 1}, type, std::vector<T> {1, 1}})
+            .expected({{2, 1, 2, 1}, element::boolean, std::vector<char> {0, 1, 0, 1}})};
+    return compParams;
+}
+
+std::vector<RefComparisonParams> generateComparisonCombinedParams() {
+    const std::vector<std::vector<RefComparisonParams>> compTypeParams {
+        generateComparisonParams<element::Type_t::f32>(element::f32),
+        generateComparisonParams<element::Type_t::f16>(element::f16),
+        generateComparisonParams<element::Type_t::i32>(element::i32),
+        generateComparisonParams<element::Type_t::u32>(element::u32),
+        generateComparisonParams<element::Type_t::u8>(element::boolean)};
+    std::vector<RefComparisonParams> combinedParams;
+
+    for (const auto& params : compTypeParams) {
+        combinedParams.insert(combinedParams.end(), params.begin(), params.end());
+    }
+    return combinedParams;
+}
+
+INSTANTIATE_TEST_SUITE_P(smoke_Comparison_With_Hardcoded_Refs, ReferenceComparisonLayerTest,
+                         ::testing::ValuesIn(generateComparisonCombinedParams()),
+                         ReferenceComparisonLayerTest::getTestCaseName);
+
+template <element::Type_t IN_ET>
+std::vector<RefComparisonParams> generateNumericParams(const element::Type& type) {
+    using T = typename element_type_traits<IN_ET>::value_type;
+    std::vector<RefComparisonParams> compParams {
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{4}, type, std::vector<T> {-2.5f, 25.5f, 2.25f, NAN}})
+            .input2({{4}, type, std::vector<T> {10.0f, 5.0f, 2.25f, 10.0f}})
+            .expected({{4}, element::boolean, std::vector<char> {1, 1, 0, 1}}),
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{2, 3}, type, std::vector<T> {0.0f, NAN, NAN, 1.0f, 21.0f, -INFINITY}})
+            .input2({{2, 3}, type, std::vector<T> {1.0f, NAN, 23.0f, 1.0f, 19.0f, 21.0f}})
+            .expected({{2, 3}, element::boolean, std::vector<char> {1, 1, 1, 0, 1, 1}}),
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{1}, type, std::vector<T> {INFINITY}})
+            .input2({{1}, type, std::vector<T> {INFINITY}})
+            .expected({{1}, element::boolean, std::vector<char> {0}}),
+        Builder {}
+            .compType(ComparisonTypes::NOT_EQUAL)
+            .input1({{5}, type, std::vector<T> {-2.5f, 25.5f, 2.25f, INFINITY, 6.0f}})
+            .input2({{5}, type, std::vector<T> {10.0f, 5.0f, 2.25f, 10.0f, -INFINITY}})
+            .expected({{5}, element::boolean, std::vector<char> {1, 1, 0, 1, 1}})};
+    return compParams;
+}
+
+std::vector<RefComparisonParams> generateNumericCombinedParams() {
+    const std::vector<std::vector<RefComparisonParams>> compTypeParams {
+        generateNumericParams<element::Type_t::f16>(element::f16),
+        generateNumericParams<element::Type_t::f32>(element::f32)};
+    std::vector<RefComparisonParams> combinedParams;
+
+    for (const auto& params : compTypeParams) {
+        combinedParams.insert(combinedParams.end(), params.begin(), params.end());
+    }
+    return combinedParams;
+}
+
+INSTANTIATE_TEST_SUITE_P(smoke_Numeric_With_Hardcoded_Refs, ReferenceComparisonLayerTest, ::testing::ValuesIn(generateNumericCombinedParams()),
+                         ReferenceComparisonLayerTest::getTestCaseName);
+} // namespace
+} // namespace ComparisonOpsRefTestDefinitions
+} // namespace reference_tests
diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/not_equal.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/not_equal.hpp
index 7b10df0e22d197..fcc3206c2921f2 100644
--- a/ngraph/core/reference/include/ngraph/runtime/reference/not_equal.hpp
+++ b/ngraph/core/reference/include/ngraph/runtime/reference/not_equal.hpp
@@ -18,17 +18,6 @@
 namespace ngraph {
 namespace runtime {
 namespace reference {
-template <typename T>
-void not_equal(const T* arg0,
-               const T* arg1,
-               char* out,
-               size_t count)  // TODO: using char for bool, is this right?
-{
-    for (size_t i = 0; i < count; i++) {
-        out[i] = arg0[i] != arg1[i];
-    }
-}
-
 template <typename T, typename U>
 void not_equal(const T* arg0,
                const T* arg1,
diff --git a/ngraph/core/src/op/not_equal.cpp b/ngraph/core/src/op/not_equal.cpp
index a97751381a68f4..cebed042fedf0c 100644
--- a/ngraph/core/src/op/not_equal.cpp
+++ b/ngraph/core/src/op/not_equal.cpp
@@ -7,6 +7,7 @@
 #include "itt.hpp"
 #include "ngraph/runtime/host_tensor.hpp"
 #include "ngraph/runtime/reference/not_equal.hpp"
+#include "ngraph/validation_util.hpp"
 
 using namespace std;
 using namespace ngraph;
@@ -65,6 +66,7 @@ shared_ptr<Node> op::v1::NotEqual::clone_with_new_inputs(const OutputVector& new
 
 bool op::v1::NotEqual::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const {
     NGRAPH_OP_SCOPE(v1_NotEqual_evaluate);
+    NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 2));
     return not_equalop::evaluate_not_equal(inputs[0], inputs[1], outputs[0], get_autob());
 }
 
diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt
index 1ac9ee9fcbad84..41abe36fed6106 100644
--- a/ngraph/test/CMakeLists.txt
+++ b/ngraph/test/CMakeLists.txt
@@ -416,7 +416,6 @@ set(MULTI_TEST_SRC
     backend/bucketize.in.cpp
     backend/builder_reduce_ops_opset1.in.cpp
     backend/ceiling.in.cpp
-    backend/comparison.in.cpp
     backend/concat.in.cpp
     backend/constant.in.cpp
     backend/convolution_backprop.in.cpp
diff --git a/ngraph/test/backend/comparison.in.cpp b/ngraph/test/backend/comparison.in.cpp
deleted file mode 100644
index dbe1247b89668c..00000000000000
--- a/ngraph/test/backend/comparison.in.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (C) 2018-2021 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#include <algorithm>
-#include <cinttypes>
-#include <cmath>
-#include <cstdlib>
-#include <random>
-#include <string>
-
-#include "gtest/gtest.h"
-#include "ngraph/log.hpp"
-#include "ngraph/ngraph.hpp"
-#include "ngraph/runtime/tensor.hpp"
-#include "runtime/backend.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
-#include "util/random.hpp"
-#include "util/test_control.hpp"
-#include "util/test_tools.hpp"
-
-using namespace std;
-using namespace ngraph;
-
-static string s_manifest = "${MANIFEST}";
-
-NGRAPH_TEST(${BACKEND_NAME}, notequal) {
-    Shape shape{2, 2, 2};
-    auto A = make_shared<op::Parameter>(element::f32, shape);
-    auto B = make_shared<op::Parameter>(element::f32, shape);
-    auto f = make_shared<Function>(make_shared<op::v1::NotEqual>(A, B), ParameterVector{A, B});
-
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    copy_data(a, vector<float>{1, 8, -8, 17, -0.5, 0, 1, 1});
-    auto b = backend->create_tensor(element::f32, shape);
-    copy_data(b, vector<float>{1, 8, 4, 8, 0, 0, 1, 1.5});
-    auto result = backend->create_tensor(element::boolean, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a, b});
-    EXPECT_EQ((vector<char>{0, 0, 1, 1, 1, 0, 0, 1}), read_vector<char>(result));
-}
diff --git a/ngraph/test/backend/zero_sized.in.cpp b/ngraph/test/backend/zero_sized.in.cpp
index 29846308d63c1b..ba8bdf53dbf57b 100644
--- a/ngraph/test/backend/zero_sized.in.cpp
+++ b/ngraph/test/backend/zero_sized.in.cpp
@@ -227,10 +227,6 @@ NGRAPH_TEST(${BACKEND_NAME}, zero_sized_multiply) {
     make_binary_empty_test<op::v1::Multiply>("${BACKEND_NAME}");
 }
 
-NGRAPH_TEST(${BACKEND_NAME}, zero_sized_not_equal) {
-    make_binary_empty_test<op::v1::NotEqual>("${BACKEND_NAME}", true);
-}
-
 NGRAPH_TEST(${BACKEND_NAME}, zero_sized_power) {
     make_binary_empty_test<op::v1::Power>("${BACKEND_NAME}");
 }
diff --git a/ngraph/test/runtime/ie/unit_test.manifest b/ngraph/test/runtime/ie/unit_test.manifest
index 8c7fad49569dda..8ac6a7d0564ff8 100644
--- a/ngraph/test/runtime/ie/unit_test.manifest
+++ b/ngraph/test/runtime/ie/unit_test.manifest
@@ -290,7 +290,6 @@ zero_sized_lesseq
 zero_sized_maximum
 zero_sized_minimum
 zero_sized_multiply
-zero_sized_not_equal
 zero_sized_power
 zero_sized_subtract
 sum_trivial