From 77cd9efc82f0a2eac1faf467a24cf55a7eb4b5f7 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Tue, 12 Mar 2024 11:15:46 +0000 Subject: [PATCH 1/9] Add fp32 -> nf4 convert test --- .../functional/op_reference/base_reference_test.cpp | 7 +++++++ .../tests/functional/op_reference/convert_like.cpp | 12 ++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp b/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp index 1f2a01b6866e1e..aca7b6b2a0aa2a 100644 --- a/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp +++ b/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp @@ -216,6 +216,13 @@ void CommonReferenceTest::ValidateBlobs(const ov::Tensor& refBlob, threshold, abs_threshold); break; + case ov::element::nf4: + ov::test::utils::compare_raw_data(static_cast(refBlob.data()), + static_cast(outBlob.data()), + actual_comparision_size / 2, + threshold, + abs_threshold); + break; case ov::element::string: ov::test::utils::compare_str(refBlob, outBlob); break; diff --git a/src/plugins/template/tests/functional/op_reference/convert_like.cpp b/src/plugins/template/tests/functional/op_reference/convert_like.cpp index 29c4d57a2160ab..30ec0291916d40 100644 --- a/src/plugins/template/tests/functional/op_reference/convert_like.cpp +++ b/src/plugins/template/tests/functional/op_reference/convert_like.cpp @@ -238,7 +238,6 @@ INSTANTIATE_TEST_SUITE_P( vector{0.5f, 1.5f, 0.5f, 2.5f, 1.5f, 0.5f, 3.5f, 2.5f, 0.5f, 0.5f, 2.5f, 0.5f, 0.5f, 0.5f, 1.5f}, std::vector< float>{0.5f, 1.5f, 0.5f, 2.5f, 1.5f, 0.5f, 3.5f, 2.5f, 0.5f, 0.5f, 2.5f, 0.5f, 0.5f, 0.5f, 1.5f}), - // destination i4 ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, @@ -1286,7 +1285,16 @@ INSTANTIATE_TEST_SUITE_P( ov::element::f32, ov::element::u64, std::vector{1, 2, 2.2, 3.8}, - std::vector{1, 2, 2, 3})), + std::vector{1, 2, 2, 3}), + // destination nf4 (use quantization) + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::f32, + ov::element::nf4, + std::vector{-0.6961928009986877f, 0.7229568362236023f, 1.0f, -0.5250730514526367f}, + std::vector{0xE1, 0x2F}, + 4, + 4)), ReferenceConversionLayerTest::getTestCaseName); } // namespace } // namespace ConversionOpsRefTestDefinitions From 11c3e3576967fdce32636910ee9d697aea2d0a4c Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Tue, 12 Mar 2024 13:02:33 +0000 Subject: [PATCH 2/9] Use element iterator in Convert --- .../openvino/core/type/element_iterator.hpp | 56 +++++++++- .../include/openvino/reference/convert.hpp | 101 ++++-------------- src/core/src/op/convert.cpp | 35 ++---- src/plugins/template/backend/ops/convert.cpp | 15 +-- 4 files changed, 86 insertions(+), 121 deletions(-) diff --git a/src/core/dev_api/openvino/core/type/element_iterator.hpp b/src/core/dev_api/openvino/core/type/element_iterator.hpp index 331bd3684e576e..231204f57dc213 100644 --- a/src/core/dev_api/openvino/core/type/element_iterator.hpp +++ b/src/core/dev_api/openvino/core/type/element_iterator.hpp @@ -5,6 +5,7 @@ #pragma once #include "openvino/core/type/element_type_traits.hpp" +#include "openvino/core/type/nf4.hpp" namespace ov { namespace util { @@ -96,6 +97,11 @@ constexpr size_t bit_width() { return 4; } +template <> +constexpr size_t bit_width() { + return 4; +} + template <> constexpr size_t bit_width() { return 4; @@ -147,6 +153,12 @@ class BitProxy> m_bit_shift) & value_mask; } + void set_bit_value(uint8_t value) { + constexpr auto value_mask = util::make_n_bit_mask(m_bits); + *m_ptr &= ~(value_mask << m_bit_shift); + *m_ptr |= value << m_bit_shift; + } + public: using value_type = typename std::decay::type; //!< Fundamental type of bound to BitProxy. @@ -182,6 +194,19 @@ class BitProxy(get_bit_value()); } + /** + * @brief Converts to float for NF4. + * + * @note Implementation aligned to ConvertNF4::unpack, de-quantization applied only when converting to + * floating point. For integral types get bit value. + * + * @return Converted NF4 value to float. + */ + template ::type* = nullptr> + operator float() const { + return ConvertNF4::dequantize(get_bit_value()); + } + /** * @brief Converts to fundamental type. * @@ -207,8 +232,17 @@ class BitProxy& operator=(const value_type v) { constexpr auto value_mask = util::make_n_bit_mask(m_bits); - *m_ptr &= ~(value_mask << m_bit_shift); - *m_ptr |= (static_cast(v) & value_mask) << m_bit_shift; + set_bit_value(static_cast(v) & value_mask); + return *this; + } + + /** + * @brief Sets current NF4 value from float using qunatization. + * @param v Value to be set. + */ + template ::type* = nullptr> + BitProxy& operator=(const float v) { + set_bit_value(ConvertNF4::quantize(v)); return *this; } }; @@ -490,7 +524,7 @@ class Iterator { * * @tparam ET Type of ov::element::Type_t. * @tparam T Type of pointer data. Must be fundamental type of ET. - + * * @param ptr Pointer to data. * @return Element iterator for type ET. */ @@ -498,5 +532,21 @@ template iterator(T* ptr) { return {ptr}; } + +/** + * @brief Make iterator from pointer for standard types. + * + * To have common interface for all ov::element::Type. Just return input pointer. + * + * @tparam ET Type of ov::element::Type_t. + * @tparam T Type of pointer data. Must be fundamental type of ET. + * + * @param ptr Pointer to data. + * @return Element iterator same as input pointer. + */ +template ::type* = nullptr> +constexpr T* iterator(T* ptr) { + return ptr; +} } // namespace element } // namespace ov diff --git a/src/core/reference/include/openvino/reference/convert.hpp b/src/core/reference/include/openvino/reference/convert.hpp index 8af21adc189cac..479e89c6f2e5b3 100644 --- a/src/core/reference/include/openvino/reference/convert.hpp +++ b/src/core/reference/include/openvino/reference/convert.hpp @@ -7,6 +7,7 @@ #include #include +#include "openvino/core/type/element_iterator.hpp" #include "openvino/core/type/element_type.hpp" #include "openvino/core/type/float16.hpp" #include "openvino/core/type/nf4.hpp" @@ -14,87 +15,6 @@ namespace ov { namespace reference { namespace detail { -inline void set_u1(uint8_t* buf, size_t idx, uint8_t val) { - const size_t byte_idx = idx / 8; - const uint8_t bit_idx = 7 - (idx % 8); // Reversed order of bits - if (val) { - buf[byte_idx] |= (1 << bit_idx); - } else { - buf[byte_idx] &= ~(1 << bit_idx); - } -} - -inline uint8_t get_u1(const uint8_t* buf, size_t idx) { - const size_t byte_idx = idx / 8; - const uint8_t bit_idx = 7 - (idx % 8); // Reversed order of bits - return (buf[byte_idx] & (1 << bit_idx)) ? 1 : 0; -} - -inline void set_u4(uint8_t* buf, size_t idx, uint8_t val) { - const size_t byte_idx = idx / 2; - const uint8_t bit_shift = 4 * (idx % 2); - buf[byte_idx] &= ~(0xF << bit_shift); // half byte zeroed - buf[byte_idx] |= ((val & 0xF) << bit_shift); // set 1's -} - -inline uint8_t get_u4(const uint8_t* buf, size_t idx) { - const size_t byte_idx = idx / 2; - const uint8_t bit_shift = 4 * (idx % 2); - return (buf[byte_idx] >> bit_shift) & 0xF; -} - -inline void set_i4(uint8_t* buf, size_t idx, int8_t val) { - const size_t byte_idx = idx / 2; - const uint8_t bit_shift = 4 * (idx % 2); - buf[byte_idx] &= ~(0xF << bit_shift); // half byte zeroed - buf[byte_idx] |= ((val & 0xF) << bit_shift); // set 1's -} - -inline int8_t get_i4(const uint8_t* buf, size_t idx) { - const size_t byte_idx = idx / 2; - const uint8_t bit_shift = 4 * (idx % 2); - uint8_t val = (buf[byte_idx] >> bit_shift) & 0xF; - if (val & 0x08) { // negative number - val |= 0xF0; - } - return val; -} -template -TO get_value(const uint8_t* buf, size_t idx, element::Type from_type) { - if (from_type == element::u1) { - return detail::get_u1(buf, idx); - } - - if (from_type == element::u4) { - return detail::get_u4(buf, idx); - } - - if (from_type == element::i4) { - return detail::get_i4(buf, idx); - } - - auto v = reinterpret_cast(buf); - return static_cast(v[idx]); -} - -template -void lp_convert(const TI* arg, TO* out, size_t count, element::Type_t src_type, element::Type_t dst_type) { - const uint8_t* input = reinterpret_cast(arg); - uint8_t* output = reinterpret_cast(out); - for (size_t i = 0; i < count; ++i) { - if (dst_type == element::u1) { - detail::set_u1(output, i, detail::get_value(input, i, src_type)); - } else if (dst_type == element::u4) { - detail::set_u4(output, i, detail::get_value(input, i, src_type)); - } else if (dst_type == element::i4) { - detail::set_i4(output, i, detail::get_value(input, i, src_type)); - } else if (src_type == element::nf4) { - ov::ConvertNF4::unpack(out, input, i); - } else { - out[i] = detail::get_value(input, i, src_type); - } - } -} template typename std::enable_if::value, TO>::type convert(const TI v) { @@ -107,6 +27,25 @@ typename std::enable_if::value, TO>::type convert(const T } } // namespace detail +template +void convert(InputIt arg, OutputIt out, const size_t count) { + using IN_T = typename std::iterator_traits::value_type; + using OUT_T = typename std::iterator_traits::value_type; + + // Deduce types for NF4 <-> floating point conversion to use quantization. + using From = + typename std::conditional>::value && + !std::is_integral::value, + const float, + IN_T>::type; + using To = typename std::conditional>::value && + !std::is_integral::value, + float, + OUT_T>::type; + + std::transform(arg, arg + count, out, detail::convert); +} + template void convert(const TI* arg, TO* out, const size_t count) { std::transform(arg, arg + count, out, detail::convert); diff --git a/src/core/src/op/convert.cpp b/src/core/src/op/convert.cpp index 0ba495058713ba..12d1f1f7a841ab 100644 --- a/src/core/src/op/convert.cpp +++ b/src/core/src/op/convert.cpp @@ -15,50 +15,31 @@ namespace ov { namespace op { namespace convert { -constexpr bool is_lp_type(const element::Type_t et) { - return (et == element::i4) || (et == element::u1) || (et == element::u4) || (et == element::nf4); -} - #define CONVERT_ET_LIST \ boolean, bf16, f16, f32, f64, i4, i8, i16, i32, i64, u1, u4, u8, u16, u32, u64, nf4, f8e4m3, f8e5m2 struct Evaluate : public element::NoAction { using element::NoAction::visit; - template > + + template > static result_type visit(const Tensor& arg, Tensor& out, const size_t count) { using namespace ov::element; return IF_TYPE_OF(Convert_out, CONVERT_ET_LIST, - EvalByOutputType, + EvalByOutputType, out.get_element_type(), - reinterpret_cast(arg.data()), + iterator(reinterpret_cast(arg.data())), out, - count, - ET); + count); } private: - template struct EvalByOutputType : public element::NoAction { using element::NoAction::visit; - template , - typename std::enable_if::type* = nullptr> - static result_type visit(const T* arg, Tensor& out, const size_t count, T_ET&& arg_et) { - reference::detail::lp_convert(arg, reinterpret_cast(out.data()), count, arg_et, ET); - return true; - } - - template , - typename std::enable_if::type* = nullptr> - static result_type visit(const T* arg, Tensor& out, const size_t count, T_ET&&) { - reference::convert(arg, out.data(), count); + template > + static result_type visit(InputIter arg, Tensor& out, const size_t count) { + reference::convert(arg, element::iterator(reinterpret_cast(out.data())), count); return true; } }; diff --git a/src/plugins/template/backend/ops/convert.cpp b/src/plugins/template/backend/ops/convert.cpp index 0bbda4f3195f36..d0ba5a3f5c8deb 100644 --- a/src/plugins/template/backend/ops/convert.cpp +++ b/src/plugins/template/backend/ops/convert.cpp @@ -5,6 +5,7 @@ #include "openvino/reference/convert.hpp" #include "evaluate_node.hpp" +#include "openvino/core/type/element_iterator.hpp" namespace convert_like_v1 { template @@ -16,13 +17,9 @@ inline void evaluate(const std::shared_ptr& op, outputs[0].set_shape(inputs[0].get_shape()); size_t element_count = ov::shape_size(outputs[0].get_shape()); - if (((ti == ov::element::u1) || (to == ov::element::u1)) || ((ti == ov::element::u4) || (to == ov::element::u4)) || - ((ti == ov::element::i4) || (to == ov::element::i4)) || - ((ti == ov::element::nf4) || (to == ov::element::nf4))) { - ov::reference::detail::lp_convert(inputs[0].data(), outputs[0].data(), element_count, ti, to); - } else { - ov::reference::convert(inputs[0].data(), outputs[0].data(), element_count); - } + ov::reference::convert(ov::element::iterator(inputs[0].data()), + ov::element::iterator(outputs[0].data()), + element_count); } } // namespace convert_like_v1 @@ -89,9 +86,7 @@ template <> bool evaluate_node(std::shared_ptr node, ov::TensorVector& outputs, const ov::TensorVector& inputs) { - auto element_type = node->get_output_element_type(0); - if (ov::is_type(node) || ov::is_type(node)) - element_type = node->get_input_element_type(1); + const auto& element_type = node->get_output_element_type(0); switch (element_type) { case ov::element::boolean: From 55372c25e972ca50d159ace09905dfdee3ec9d52 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Wed, 13 Mar 2024 10:29:09 +0000 Subject: [PATCH 3/9] Update tensor byte size calculation --- src/core/src/runtime/itensor.cpp | 18 ++++++++-- src/core/tests/ov_tensor_test.cpp | 40 +++++++++++++++++++-- src/core/tests/tensor.cpp | 50 ++++++++++++++++++++++++++- src/inference/src/dev/make_tensor.cpp | 33 +++++++++++++++--- 4 files changed, 130 insertions(+), 11 deletions(-) diff --git a/src/core/src/runtime/itensor.cpp b/src/core/src/runtime/itensor.cpp index 29dcde3eefb380..e281e84ca9caf1 100644 --- a/src/core/src/runtime/itensor.cpp +++ b/src/core/src/runtime/itensor.cpp @@ -2,14 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "openvino/runtime/itensor.hpp" - #include #include "openvino/core/except.hpp" #include "openvino/core/shape_util.hpp" +#include "openvino/core/type/element_iterator.hpp" #include "openvino/runtime/allocator.hpp" #include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/properties.hpp" namespace ov { @@ -21,7 +21,19 @@ size_t ITensor::get_size() const { } size_t ITensor::get_byte_size() const { - return (get_size() * get_element_type().bitwidth() + 8 - 1) / 8; + const auto& et = get_element_type(); + auto byte_size = get_size() * et.bitwidth(); + if (element::is_split_bit_type(et)) { + constexpr size_t storage_unit_size = 24; + byte_size += storage_unit_size - 1; + byte_size /= storage_unit_size; + byte_size *= 3; + } else { + constexpr size_t storage_unit_size = 8; + byte_size += storage_unit_size - 1; + byte_size /= storage_unit_size; + } + return byte_size; } bool ITensor::is_continuous() const { diff --git a/src/core/tests/ov_tensor_test.cpp b/src/core/tests/ov_tensor_test.cpp index 34daa178e6f893..76ed5caaecae44 100644 --- a/src/core/tests/ov_tensor_test.cpp +++ b/src/core/tests/ov_tensor_test.cpp @@ -22,6 +22,7 @@ #include "openvino/runtime/tensor.hpp" using OVTensorTest = ::testing::Test; +using testing::_; const size_t string_size = ov::element::string.size(); @@ -186,16 +187,49 @@ struct OVMockAllocator { }; TEST_F(OVTensorTest, canCreateTensorUsingMockAllocator) { + constexpr size_t exp_size = 24; ov::Shape shape = {1, 2, 3}; OVMockAllocator allocator; - EXPECT_CALL(*allocator.impl, allocate(::testing::_, ::testing::_)) - .WillRepeatedly(testing::Return(reinterpret_cast(1))); - EXPECT_CALL(*allocator.impl, deallocate(::testing::_, ::testing::_, ::testing::_)).Times(1); + EXPECT_CALL(*allocator.impl, allocate(exp_size, _)).WillRepeatedly(testing::Return(reinterpret_cast(1))); + EXPECT_CALL(*allocator.impl, deallocate(_, exp_size, _)).Times(1); { ov::Tensor t{ov::element::f32, shape, allocator}; } } +TEST_F(OVTensorTest, canCreateTensorU2UsingMockAllocator) { + constexpr size_t exp_size = 2; + ov::Shape shape = {1, 2, 3}; + OVMockAllocator allocator; + + EXPECT_CALL(*allocator.impl, allocate(exp_size, _)).WillRepeatedly(testing::Return(reinterpret_cast(1))); + EXPECT_CALL(*allocator.impl, deallocate(_, exp_size, _)).Times(1); + + { ov::Tensor t{ov::element::u2, shape, allocator}; } +} + +TEST_F(OVTensorTest, canCreateTensorU3UsingMockAllocator) { + constexpr size_t exp_size = 3; + ov::Shape shape = {1, 2, 3}; + OVMockAllocator allocator; + + EXPECT_CALL(*allocator.impl, allocate(exp_size, _)).WillRepeatedly(testing::Return(reinterpret_cast(1))); + EXPECT_CALL(*allocator.impl, deallocate(_, exp_size, _)).Times(1); + + { ov::Tensor t{ov::element::u3, shape, allocator}; } +} + +TEST_F(OVTensorTest, canCreateTensorU6UsingMockAllocator) { + constexpr size_t exp_size = 6; + ov::Shape shape = {1, 2, 3}; + OVMockAllocator allocator; + + EXPECT_CALL(*allocator.impl, allocate(exp_size, _)).WillRepeatedly(testing::Return(reinterpret_cast(1))); + EXPECT_CALL(*allocator.impl, deallocate(_, exp_size, _)).Times(1); + + { ov::Tensor t{ov::element::u6, shape, allocator}; } +} + TEST_F(OVTensorTest, canAccessExternalData) { ov::Shape shape = {1, 1, 3}; float data[] = {5.f, 6.f, 7.f}; diff --git a/src/core/tests/tensor.cpp b/src/core/tests/tensor.cpp index 5b19926758472d..319fa6c86b741e 100644 --- a/src/core/tests/tensor.cpp +++ b/src/core/tests/tensor.cpp @@ -15,8 +15,9 @@ #include "openvino/op/relu.hpp" using namespace std; -using namespace ov; +namespace ov { +namespace test { TEST(tensor, tensor_names) { auto arg0 = make_shared(element::f32, Shape{1}); arg0->set_friendly_name("data"); @@ -42,3 +43,50 @@ TEST(tensor, create_tensor_with_zero_dims_check_stride) { EXPECT_EQ(stride.back(), 0); EXPECT_EQ(tensor.is_continuous(), true); } + +TEST(tensor, get_byte_size_u2_less_than_min_storage_unit) { + const auto tensor = Tensor(element::u2, Shape{3}); + EXPECT_EQ(tensor.get_byte_size(), 1); +} + +TEST(tensor, get_byte_size_u2_even_div_by_storage_unit) { + const auto tensor = Tensor(element::u2, Shape{16}); + EXPECT_EQ(tensor.get_byte_size(), 4); +} + +TEST(tensor, get_byte_size_u2_not_even_div_by_storage_unit) { + const auto tensor = Tensor(element::u2, Shape{17}); + EXPECT_EQ(tensor.get_byte_size(), 5); +} + +TEST(tensor, get_byte_size_u3_less_than_min_storage_unit) { + const auto tensor = Tensor(element::u3, Shape{3}); + EXPECT_EQ(tensor.get_byte_size(), 3); +} + +TEST(tensor, get_byte_size_u3_even_div_by_storage_unit) { + const auto tensor = Tensor(element::u3, Shape{16}); + EXPECT_EQ(tensor.get_byte_size(), 2 * 3); +} + +TEST(tensor, get_byte_size_u3_not_even_div_by_storage_unit) { + const auto tensor = Tensor(element::u3, Shape{17}); + EXPECT_EQ(tensor.get_byte_size(), 3 + 2 * 3); +} + +TEST(tensor, get_byte_size_u6_less_than_min_storage_unit) { + const auto tensor = Tensor(element::u6, Shape{3}); + EXPECT_EQ(tensor.get_byte_size(), 3); +} + +TEST(tensor, get_byte_size_u6_even_div_by_storage_unit) { + const auto tensor = Tensor(element::u6, Shape{16}); + EXPECT_EQ(tensor.get_byte_size(), 4 * 3); +} + +TEST(tensor, get_byte_size_u6_not_even_div_by_storage_unit) { + const auto tensor = Tensor(element::u6, Shape{17}); + EXPECT_EQ(tensor.get_byte_size(), 3 + 4 * 3); +} +} // namespace test +} // namespace ov diff --git a/src/inference/src/dev/make_tensor.cpp b/src/inference/src/dev/make_tensor.cpp index 98c78dfc06c34c..234f2787d137e4 100644 --- a/src/inference/src/dev/make_tensor.cpp +++ b/src/inference/src/dev/make_tensor.cpp @@ -7,6 +7,7 @@ #include #include +#include "openvino/core/type/element_iterator.hpp" #include "openvino/runtime/iremote_tensor.hpp" #include "openvino/runtime/properties.hpp" #include "openvino/runtime/tensor.hpp" @@ -37,6 +38,30 @@ Shape make_roi_shape(const Shape& tensor_shape, const Coordinate& begin, const C return roi_shape; } + +/** + * @brief Gets size of elements count and given precision in bytes. + * + * @param num_elements Number of elements. + * @param element_type Elements precision. + * + * @return Elements size in bytes. + */ +size_t get_elements_byte_size(const size_t num_elements, const element::Type& element_type) { + auto byte_size = num_elements * element_type.bitwidth(); + if (element::is_split_bit_type(element_type)) { + constexpr size_t storage_unit_size = 24; + byte_size += storage_unit_size - 1; + byte_size /= storage_unit_size; + byte_size *= 3; + } else { + constexpr size_t storage_unit_size = 8; + byte_size += storage_unit_size - 1; + byte_size /= storage_unit_size; + } + return byte_size; +} + } // namespace /** @@ -200,10 +225,10 @@ class AllocatedTensor : public ViewTensor { AllocatedTensor(const element::Type element_type, const Shape& shape, const Allocator& allocator) : ViewTensor{element_type, shape, - [&] { + [this, &shape, &element_type, &allocator] { OPENVINO_ASSERT(allocator, "Allocator was not initialized"); - auto num_elements = shape_size(shape); - auto data = const_cast(allocator).allocate(element_type.size() * num_elements); + const auto byte_size = get_elements_byte_size(shape_size(shape), element_type); + auto data = const_cast(allocator).allocate(byte_size); initialize_elements(data, element_type, shape); return data; }()}, @@ -263,7 +288,7 @@ class AllocatedTensor : public ViewTensor { } size_t get_bytes_capacity() const { - return (get_capacity() * get_element_type().bitwidth() + 8 - 1) / 8; + return get_elements_byte_size(get_capacity(), get_element_type()); } Allocator m_allocator; From 232f9f2e1230ae18690aa644cd6a8c0e6d0303fb Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Wed, 13 Mar 2024 10:31:57 +0000 Subject: [PATCH 4/9] Add u2, u3, u6 type to Convert --- src/core/src/op/convert.cpp | 2 +- .../op_reference/base_reference_test.cpp | 21 + .../functional/op_reference/convert_like.cpp | 776 +++++++++++++++++- 3 files changed, 797 insertions(+), 2 deletions(-) diff --git a/src/core/src/op/convert.cpp b/src/core/src/op/convert.cpp index 12d1f1f7a841ab..388bd3597a4356 100644 --- a/src/core/src/op/convert.cpp +++ b/src/core/src/op/convert.cpp @@ -16,7 +16,7 @@ namespace op { namespace convert { #define CONVERT_ET_LIST \ - boolean, bf16, f16, f32, f64, i4, i8, i16, i32, i64, u1, u4, u8, u16, u32, u64, nf4, f8e4m3, f8e5m2 + boolean, bf16, f16, f32, f64, i4, i8, i16, i32, i64, u1, u2, u3, u4, u6, u8, u16, u32, u64, nf4, f8e4m3, f8e5m2 struct Evaluate : public element::NoAction { using element::NoAction::visit; diff --git a/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp b/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp index aca7b6b2a0aa2a..715ab1fb8f4d92 100644 --- a/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp +++ b/src/plugins/template/tests/functional/op_reference/base_reference_test.cpp @@ -216,6 +216,27 @@ void CommonReferenceTest::ValidateBlobs(const ov::Tensor& refBlob, threshold, abs_threshold); break; + case ov::element::u2: + ov::test::utils::compare_raw_data(static_cast(refBlob.data()), + static_cast(outBlob.data()), + actual_comparision_size / 4, + threshold, + abs_threshold); + break; + case ov::element::u3: + ov::test::utils::compare_raw_data(static_cast(refBlob.data()), + static_cast(outBlob.data()), + 3 * (actual_comparision_size / 8), + threshold, + abs_threshold); + break; + case ov::element::u6: + ov::test::utils::compare_raw_data(static_cast(refBlob.data()), + static_cast(outBlob.data()), + 3 * (actual_comparision_size / 4), + threshold, + abs_threshold); + break; case ov::element::nf4: ov::test::utils::compare_raw_data(static_cast(refBlob.data()), static_cast(outBlob.data()), diff --git a/src/plugins/template/tests/functional/op_reference/convert_like.cpp b/src/plugins/template/tests/functional/op_reference/convert_like.cpp index 30ec0291916d40..60462c51e9ecf1 100644 --- a/src/plugins/template/tests/functional/op_reference/convert_like.cpp +++ b/src/plugins/template/tests/functional/op_reference/convert_like.cpp @@ -21,6 +21,27 @@ INSTANTIATE_TEST_SUITE_P( ReferenceConversionLayerTest, ::testing::Values( // destination boolean + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::boolean, + std::vector{0b10010011}, + std::vector{1, 1, 0, 1}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::boolean, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 1, 0, 1, 0, 0, 0, 1}, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::boolean, + std::vector{0x21, 0x03, 0x00}, + std::vector{1, 1, 0, 1}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{2, 3}, ov::element::u8, @@ -58,6 +79,27 @@ INSTANTIATE_TEST_SUITE_P( -std::numeric_limits::infinity()}, std::vector{0, 1, 1, 0, 1, 1, 1, 1, 1}), // destination bf16 + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::bf16, + std::vector{0b10010011}, + std::vector{2.0f, 1.0f, 0.0f, 3.0f}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::bf16, + std::vector{0x21, 0x03, 0x00}, + std::vector{0.0f, 2.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::bf16, + std::vector{0x21, 0x03, 0x00}, + std::vector{2.0f, 1.0f, 0.0f, 3.0f}, + 4), ConvertParams( ConversionTypes::CONVERT_LIKE, ov::PartialShape{1, 1, 3, 5}, @@ -75,6 +117,27 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0, 10, 15, 20, 43, 56, 78, 99, 102, 130, 142}), // destination f16 + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::f16, + std::vector{0b10010011}, + std::vector{2.0f, 1.0f, 0.0f, 3.0f}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::f16, + std::vector{0x21, 0x03, 0x00}, + std::vector{0.0f, 2.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::f16, + std::vector{0x21, 0x03, 0x00}, + std::vector{2.0f, 1.0f, 0.0f, 3.0f}, + 4), ConvertParams( ConversionTypes::CONVERT_LIKE, ov::PartialShape{1, 1, 3, 5}, @@ -99,6 +162,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0xA0}, std::vector{1.0f, 0.0f, 1.0f, 0.0f}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::f32, + std::vector{0b10010011}, + std::vector{2.0f, 1.0f, 0.0f, 3.0f}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::f32, + std::vector{0x21, 0x03, 0x00}, + std::vector{0.0f, 2.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{2, 2}, ov::element::u4, @@ -106,6 +183,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0xBF, 0xA0}, std::vector{15.0f, 11.0f, 0.0f, 10.0f}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::f32, + std::vector{0x21, 0x03, 0x00}, + std::vector{2.0f, 1.0f, 0.0f, 3.0f}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{2, 2}, ov::element::u8, @@ -247,6 +331,22 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x01, 0x01}, 4, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::i4, + std::vector{0b10010011}, + std::vector{0x12, 0x30}, + 4, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::i4, + std::vector{0x21, 0x03, 0x00}, + std::vector{0x20, 0x10, 0x00, 0x30}, + 8, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -255,6 +355,14 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x03}, 4, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::i4, + std::vector{0x21, 0x03, 0x00}, + std::vector{0x12, 0x30}, + 4, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -359,6 +467,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::i8, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::i8, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -366,6 +488,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::i8, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -447,6 +576,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::i16, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::i16, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -454,6 +597,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::i16, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -535,6 +685,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::i32, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::i32, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -542,6 +706,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::i32, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -623,6 +794,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::i64, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::i64, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -630,6 +815,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::i64, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -713,6 +905,21 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0xA0}, 8, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u2, + ov::element::u1, + std::vector{0b10010011, 0x00}, + std::vector{0b01010000}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::u1, + std::vector{0x21, 0x03, 0x00}, + std::vector{0b00010001}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{8}, ov::element::u4, @@ -721,6 +928,14 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x90}, 8, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u6, + ov::element::u1, + std::vector{0x21, 0x03, 0x00, 0x00, 0x00, 0x00}, + std::vector{0b01010000}, + 8, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{8}, ov::element::u8, @@ -827,6 +1042,22 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x01, 0x01}, 4, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::u4, + std::vector{0b10010011}, + std::vector{0x12, 0x30}, + 4, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::u4, + std::vector{0x21, 0x03, 0x00}, + std::vector{0x20, 0x10, 0x00, 0x30}, + 8, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -835,6 +1066,14 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x03}, 4, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::u4, + std::vector{0x21, 0x03, 0x00}, + std::vector{0x12, 0x30}, + 4, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -940,6 +1179,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::u8, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::u8, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -947,6 +1200,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::u8, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -1029,6 +1289,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::u16, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::u16, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -1036,6 +1310,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::u16, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -1118,6 +1399,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::u32, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::u32, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -1125,6 +1420,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::u32, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -1206,6 +1508,20 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x81}, std::vector{1, 0, 0, 0, 0, 0, 0, 1}, 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u2, + ov::element::u64, + std::vector{0b10010011}, + std::vector{2, 1, 0, 3}, + 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{8}, + ov::element::u3, + ov::element::u64, + std::vector{0x21, 0x03, 0x00}, + std::vector{0, 2, 0, 1, 0, 0, 0, 3}, + 8), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u4, @@ -1213,6 +1529,13 @@ INSTANTIATE_TEST_SUITE_P( std::vector{0x12, 0x34}, std::vector{2, 1, 4, 3}, 4), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{4}, + ov::element::u6, + ov::element::u64, + std::vector{0x21, 0x03, 0x00}, + std::vector{2, 1, 0, 3}, + 4), ConvertParams(ConversionTypes::CONVERT_LIKE, ov::PartialShape{4}, ov::element::u8, @@ -1294,7 +1617,458 @@ INSTANTIATE_TEST_SUITE_P( std::vector{-0.6961928009986877f, 0.7229568362236023f, 1.0f, -0.5250730514526367f}, std::vector{0xE1, 0x2F}, 4, - 4)), + 4), + // destination u2 + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::boolean, + ov::element::u2, + std::vector{1, 0, 1, 1, 0, 1, 0, 0}, + std::vector{0b01000101, 0b00010000}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u1, + ov::element::u2, + std::vector{0b11001011}, + std::vector{0x50, 0x45}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{1, 8}, + ov::element::u3, + ov::element::u2, + std::vector{0b00100111, 0b00011010, 0b00001000}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u4, + ov::element::u2, + std::vector{0x21, 0x01, 0x31, 0xff}, + std::vector{0b01100100, 0b01111111}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{1, 8}, + ov::element::u6, + ov::element::u2, + std::vector{0x02, 0x13, 0x00, 0x81, 0x04, 0b01001011}, + std::vector{0b00100111, 0b00010000}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u8, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u16, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u32, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u64, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i4, + ov::element::u2, + std::vector{0x21, 0x01, 0x31, 0xff}, + std::vector{0b01100100, 0b01111111}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i8, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i16, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i32, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i64, + ov::element::u2, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{1, 8}, + ov::element::f8e4m3, + ov::element::u2, + std::vector{0.1f, 2.0f, 1.1f, 3.0f, 4.0f, 1.0f, 2.2f, 2.8f}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f16, + ov::element::u2, + std::vector{0.1f, 2.0f, 1.1f, 3.0f, 4.0f, 1.0f, 2.2f, 2.8f}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f32, + ov::element::u2, + std::vector{0.1f, 2.0f, 1.1f, 3.0f, 4.0f, 1.0f, 2.2f, 2.8f}, + std::vector{0b00100111, 0b00011010}, + 8, + 8), + // destination u3 + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::boolean, + ov::element::u3, + std::vector{1, 0, 1, 1, 0, 1, 0, 0}, + std::vector{0b01000101, 0b00010000, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u1, + ov::element::u3, + std::vector{0b11001011}, + std::vector{0b01010000, 0b01000101, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{1, 8}, + ov::element::u2, + ov::element::u3, + std::vector{0b00100111, 0b00011010}, + std::vector{0b00100111, 0b00011010, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u4, + ov::element::u3, + std::vector{0x21, 0x01, 0x31, 0xff}, + std::vector{0b01100100, 0b01111111, 0b00000011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{1, 8}, + ov::element::u6, + ov::element::u3, + std::vector{0x02, 0x13, 0x00, 0x41, 0x26, 0x00}, + std::vector{0b00100111, 0b00011010, 0b00001001}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u8, + ov::element::u3, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010, 0b00001000}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u16, + ov::element::u3, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010, 0b00001000}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u32, + ov::element::u3, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010, 0b00001000}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u64, + ov::element::u3, + std::vector{0, 2, 1, 3, 4, 1, 2, 2}, + std::vector{0b00100111, 0b00011010, 0b00001000}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i8, + ov::element::u3, + std::vector{0, 2, 1, -3, 4, -1, 8, 7}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i16, + ov::element::u3, + std::vector{0, 2, 1, -3, 4, -1, 8, 7}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i32, + ov::element::u3, + std::vector{0, 2, 1, -3, 4, -1, 8, 7}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i64, + ov::element::u3, + std::vector{0, 2, 1, -3, 4, -1, 8, 7}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f8e4m3, + ov::element::u3, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 4.0f, -1.0f, 8.0f, 7.2f}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f8e5m2, + ov::element::u3, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 4.0f, -1.0f, 8.0f, 7.2f}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f16, + ov::element::u3, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 4.0f, -1.0f, 8.0f, 7.2f}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::bf16, + ov::element::u3, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 4.0f, -1.0f, 8.0f, 7.2f}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f32, + ov::element::u3, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 4.0f, -1.0f, 8.0f, 7.2f}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f64, + ov::element::u3, + std::vector{0.0, 2.1, 1.7, -3.1, 4.0, -1.0, 8.0, 7.2}, + std::vector{0b00100101, 0b00110011, 0b00011101}, + 8, + 8), + // destination u6 + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::boolean, + ov::element::u6, + std::vector{1, 0, 1, 1, 0, 1, 0, 0}, + std::vector{0x10, 0x11, 0x00, 0x01, 0x00, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u1, + ov::element::u6, + std::vector{0b11001011}, + std::vector{0x11, 0x00, 0x00, 0x10, 0x11, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{1, 8}, + ov::element::u2, + ov::element::u6, + std::vector{0b00100111, 0b00010000}, + std::vector{0x02, 0x13, 0x00, 0x01, 0x00, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{1, 8}, + ov::element::u3, + ov::element::u6, + std::vector{0b00100111, 0b00011010, 0b00001001}, + std::vector{0x02, 0x13, 0x00, 0x41, 0x26, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u4, + ov::element::u6, + std::vector{0x21, 0x01, 0x31, 0xff}, + std::vector{0x12, 0x10, 0x00, 0x13, 0xff, 0x00}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u8, + ov::element::u6, + std::vector{0, 2, 1, 3, 24, 1, 32, 52}, + std::vector{0x02, 0x13, 0x00, 0x81, 0x04, 0b01001011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u16, + ov::element::u6, + std::vector{0, 2, 1, 3, 24, 1, 32, 52}, + std::vector{0x02, 0x13, 0x00, 0x81, 0x04, 0b01001011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u32, + ov::element::u6, + std::vector{0, 2, 1, 3, 24, 1, 32, 52}, + std::vector{0x02, 0x13, 0x00, 0x81, 0x04, 0b01001011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::u64, + ov::element::u6, + std::vector{0, 2, 1, 3, 24, 1, 32, 52}, + std::vector{0x02, 0x13, 0x00, 0x81, 0x04, 0b01001011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i4, + ov::element::u6, + std::vector{0x21, 0x01, 0x31, 0xff}, + std::vector{0x12, 0x10, 0x00, 0x13, 0xff, 0b00001111}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i8, + ov::element::u6, + std::vector{0, 2, 1, -3, 24, -1, 32, 52}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i16, + ov::element::u6, + std::vector{0, 2, 1, -3, 24, -1, 32, 52}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i32, + ov::element::u6, + std::vector{0, 2, 1, -3, 24, -1, 32, 52}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::i64, + ov::element::u6, + std::vector{0, 2, 1, -3, 24, -1, 32, 52}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f8e4m3, + ov::element::u6, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 24.0f, -1.0f, 32.0f, 52.0f}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f8e5m2, + ov::element::u6, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 24.0f, -1.0f, 32.0f, 56.0f}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x08, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f16, + ov::element::u6, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 24.0f, -1.0f, 32.0f, 52.0f}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::bf16, + ov::element::u6, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 24.0f, -1.0f, 32.0f, 52.0f}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f32, + ov::element::u6, + std::vector{0.0f, 2.1f, 1.7f, -3.1f, 24.0f, -1.0f, 32.0f, 52.0f}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8), + ConvertParams(ConversionTypes::CONVERT_LIKE, + ov::PartialShape{2, 4}, + ov::element::f64, + ov::element::u6, + std::vector{0.0, 2.1, 1.7, -3.1, 24.0, -1.0, 32.0, 52.0}, + std::vector{0x02, 0x1d, 0b00000011, 0x8f, 0x04, 0b01111011}, + 8, + 8)), ReferenceConversionLayerTest::getTestCaseName); } // namespace } // namespace ConversionOpsRefTestDefinitions From e86a54542ca54702a2e5de7151c66db5ac9a5807 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Thu, 14 Mar 2024 09:11:06 +0000 Subject: [PATCH 5/9] Add helpers to create iterator from void pointer Improve binary size reduction --- .../openvino/core/type/element_iterator.hpp | 28 +++++++++++++++++++ src/core/src/op/convert.cpp | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/core/dev_api/openvino/core/type/element_iterator.hpp b/src/core/dev_api/openvino/core/type/element_iterator.hpp index 231204f57dc213..dfbe6e1d3687cd 100644 --- a/src/core/dev_api/openvino/core/type/element_iterator.hpp +++ b/src/core/dev_api/openvino/core/type/element_iterator.hpp @@ -548,5 +548,33 @@ template > +constexpr auto iterator(void* ptr) -> decltype(iterator(reinterpret_cast(ptr))) { + return iterator(reinterpret_cast(ptr)); +} + +/** + * @brief Make iterator from constant void pointer. + * + * Data will be reinterpreted using fundamental type for ov::element::Type. + * + * @tparam ET OpenVINO element type. + * @param ptr Pointer to data. + * @return Iterator for given ET. + */ +template >::type> +constexpr auto iterator(const void* ptr) -> decltype(iterator(reinterpret_cast(ptr))) { + return iterator(reinterpret_cast(ptr)); +} } // namespace element } // namespace ov diff --git a/src/core/src/op/convert.cpp b/src/core/src/op/convert.cpp index 388bd3597a4356..7481be24a99945 100644 --- a/src/core/src/op/convert.cpp +++ b/src/core/src/op/convert.cpp @@ -28,7 +28,7 @@ struct Evaluate : public element::NoAction { CONVERT_ET_LIST, EvalByOutputType, out.get_element_type(), - iterator(reinterpret_cast(arg.data())), + iterator(arg.data()), out, count); } @@ -39,7 +39,7 @@ struct Evaluate : public element::NoAction { template > static result_type visit(InputIter arg, Tensor& out, const size_t count) { - reference::convert(arg, element::iterator(reinterpret_cast(out.data())), count); + reference::convert(arg, element::iterator(out.data()), count); return true; } }; From 0ee0ef650c960727e84e993e69fef0b9b30b30b4 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Sat, 16 Mar 2024 15:42:51 +0000 Subject: [PATCH 6/9] Fix capture list in AllocatedTensor ctor --- src/inference/src/dev/make_tensor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inference/src/dev/make_tensor.cpp b/src/inference/src/dev/make_tensor.cpp index 234f2787d137e4..4cf2459508b3e2 100644 --- a/src/inference/src/dev/make_tensor.cpp +++ b/src/inference/src/dev/make_tensor.cpp @@ -225,7 +225,7 @@ class AllocatedTensor : public ViewTensor { AllocatedTensor(const element::Type element_type, const Shape& shape, const Allocator& allocator) : ViewTensor{element_type, shape, - [this, &shape, &element_type, &allocator] { + [&shape, &element_type, &allocator] { OPENVINO_ASSERT(allocator, "Allocator was not initialized"); const auto byte_size = get_elements_byte_size(shape_size(shape), element_type); auto data = const_cast(allocator).allocate(byte_size); From 23a0e32ce3a1b153fb10f91f0d85dfeb54fd0283 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Sun, 17 Mar 2024 07:27:06 +0000 Subject: [PATCH 7/9] Correct NF4 <-> floating point deduction --- .../include/openvino/reference/convert.hpp | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/core/reference/include/openvino/reference/convert.hpp b/src/core/reference/include/openvino/reference/convert.hpp index 479e89c6f2e5b3..19872a427f17f5 100644 --- a/src/core/reference/include/openvino/reference/convert.hpp +++ b/src/core/reference/include/openvino/reference/convert.hpp @@ -13,6 +13,15 @@ #include "openvino/core/type/nf4.hpp" namespace ov { + +template +constexpr bool is_nf4_iterator() { + using it = typename std::decay::type; + using T = fundamental_type_for; + return std::is_same>::value || + std::is_same>::value; +} + namespace reference { namespace detail { @@ -33,15 +42,10 @@ void convert(InputIt arg, OutputIt out, const size_t count) { using OUT_T = typename std::iterator_traits::value_type; // Deduce types for NF4 <-> floating point conversion to use quantization. - using From = - typename std::conditional>::value && - !std::is_integral::value, - const float, - IN_T>::type; - using To = typename std::conditional>::value && - !std::is_integral::value, - float, - OUT_T>::type; + using From = typename std:: + conditional() && !std::is_integral::value, const float, IN_T>::type; + using To = + typename std::conditional() && !std::is_integral::value, float, OUT_T>::type; std::transform(arg, arg + count, out, detail::convert); } @@ -69,7 +73,7 @@ void convert(const float16* arg, int8_t* out, size_t count); // Count how many f32 values is out of normal finite numbers range when converted to f16 size_t count_out_of_f16_range(const float* arg, size_t count); -// Convert values from f32 to f16 with claming to f16 min/max when value is out of normal finite numbers range +// Convert values from f32 to f16 with clamping to f16 min/max when value is out of normal finite numbers range void convert_from_f32_to_f16_with_clamp(const float* arg, float16* out, size_t count); } // namespace reference } // namespace ov From 5ae1dd78886b5bf0ec37a773c334a4c58fb9a357 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Tue, 19 Mar 2024 07:29:09 +0000 Subject: [PATCH 8/9] Restore removed include --- src/core/src/runtime/itensor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/src/runtime/itensor.cpp b/src/core/src/runtime/itensor.cpp index e281e84ca9caf1..851f58b403162d 100644 --- a/src/core/src/runtime/itensor.cpp +++ b/src/core/src/runtime/itensor.cpp @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "openvino/runtime/itensor.hpp" + #include #include "openvino/core/except.hpp" From e989dbd35fc051cad43d68e2835cae247f5a1ca1 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Mon, 25 Mar 2024 11:33:40 +0000 Subject: [PATCH 9/9] Fix cast of input tensor data --- src/core/src/op/convert.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/src/op/convert.cpp b/src/core/src/op/convert.cpp index 7481be24a99945..7df9e095f4c55e 100644 --- a/src/core/src/op/convert.cpp +++ b/src/core/src/op/convert.cpp @@ -28,7 +28,7 @@ struct Evaluate : public element::NoAction { CONVERT_ET_LIST, EvalByOutputType, out.get_element_type(), - iterator(arg.data()), + iterator(reinterpret_cast(arg.data())), out, count); }