From 8141baea5a69dcb2fb19f90a6b8f9e2d5bd8910f Mon Sep 17 00:00:00 2001 From: eshoguli Date: Fri, 13 Oct 2023 13:16:13 +0100 Subject: [PATCH] [CPU] Bitwise operations --- src/plugins/intel_cpu/src/cpu_types.cpp | 8 +++ src/plugins/intel_cpu/src/cpu_types.h | 4 ++ .../src/emitters/x64/jit_eltwise_emitters.cpp | 46 +++++++++++++++++ .../src/emitters/x64/jit_eltwise_emitters.hpp | 21 ++++++++ src/plugins/intel_cpu/src/nodes/eltwise.cpp | 50 +++++++++++++++---- .../instances/common/eltwise.cpp | 28 +++++++++++ .../ov_helpers/ov_models/src/eltwise.cpp | 10 ++++ .../include/common_test_utils/test_enums.hpp | 6 ++- .../common_test_utils/src/test_enums.cpp | 12 +++++ 9 files changed, 175 insertions(+), 10 deletions(-) diff --git a/src/plugins/intel_cpu/src/cpu_types.cpp b/src/plugins/intel_cpu/src/cpu_types.cpp index 03fbe1a9923b7a..be45bcc528d302 100644 --- a/src/plugins/intel_cpu/src/cpu_types.cpp +++ b/src/plugins/intel_cpu/src/cpu_types.cpp @@ -70,6 +70,10 @@ static const TypeToNameMap& get_type_to_name_tbl() { { "SoftSign", Type::Eltwise }, { "Select", Type::Eltwise}, { "Log", Type::Eltwise }, + { "BitwiseAnd", Type::Eltwise }, + { "BitwiseNot", Type::Eltwise }, + { "BitwiseOr", Type::Eltwise }, + { "BitwiseXor", Type::Eltwise }, { "Reshape", Type::Reshape }, { "Squeeze", Type::Reshape }, { "Unsqueeze", Type::Reshape }, @@ -384,6 +388,10 @@ std::string algToString(const Algorithm alg) { CASE(EltwiseErf); CASE(EltwiseSoftSign); CASE(EltwiseLog); + CASE(EltwiseBitwiseAnd); + CASE(EltwiseBitwiseNot); + CASE(EltwiseBitwiseOr); + CASE(EltwiseBitwiseXor); CASE(FQCommon); CASE(FQQuantization); CASE(FQBinarization); diff --git a/src/plugins/intel_cpu/src/cpu_types.h b/src/plugins/intel_cpu/src/cpu_types.h index 403ed62d482f8b..4ee28354899f6b 100644 --- a/src/plugins/intel_cpu/src/cpu_types.h +++ b/src/plugins/intel_cpu/src/cpu_types.h @@ -181,6 +181,10 @@ enum class Algorithm { EltwiseErf, EltwiseSoftSign, EltwiseLog, + EltwiseBitwiseAnd, + EltwiseBitwiseNot, + EltwiseBitwiseOr, + EltwiseBitwiseXor, // FakeQuantize algorithms FQCommon, diff --git a/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.cpp index d6ccd2de563227..e3d49ae12501de 100644 --- a/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.cpp +++ b/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.cpp @@ -2243,5 +2243,51 @@ void jit_select_emitter::emit_isa(const std::vector &in_vec_idxs, const h->vblendmps(vmm_dst | k_mask, vmm_src1, vmm_src0); } } + +/// LOGICAL_AND /// +jit_bitwise_and_emitter::jit_bitwise_and_emitter(x64::jit_generator *host, x64::cpu_isa_t host_isa, const std::shared_ptr& node, Precision exec_prc) +: jit_emitter(host, host_isa, exec_prc) { + prepare_table(); +} +jit_bitwise_and_emitter::jit_bitwise_and_emitter(x64::jit_generator *host, x64::cpu_isa_t host_isa, Precision exec_prc) +: jit_emitter(host, host_isa, exec_prc) { + prepare_table(); +} + +size_t jit_bitwise_and_emitter::get_inputs_num() const { return 2; } + +std::set> jit_bitwise_and_emitter::get_supported_precisions(const std::shared_ptr& node) { + return { + {element::i8, element::i8}, + {element::i16, element::i16}, + {element::i32, element::i32}, + {element::u8, element::u8}, + {element::u16, element::u16}, + {element::u32, element::u32} + }; +} + +void jit_bitwise_and_emitter::emit_impl(const std::vector& in_vec_idxs, const std::vector& out_vec_idxs) const { + if (host_isa_ == x64::sse41) { + emit_isa(in_vec_idxs, out_vec_idxs); + } else if (host_isa_ == x64::avx2) { + emit_isa(in_vec_idxs, out_vec_idxs); + } else if (host_isa_ == x64::avx512_core) { + emit_isa(in_vec_idxs, out_vec_idxs); + } else { + assert(!"unsupported isa"); + } +} + +template +void jit_bitwise_and_emitter::emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const { +} + +void jit_bitwise_and_emitter::register_table_entries() { +} + +size_t jit_bitwise_and_emitter::aux_vecs_count() const { + return 3; +} } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.hpp index e624b0f95294c0..02671abba95be5 100644 --- a/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.hpp +++ b/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.hpp @@ -669,5 +669,26 @@ class jit_select_emitter : public jit_emitter { template void emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const; }; + +class jit_bitwise_and_emitter : public jit_emitter { +public: + jit_bitwise_and_emitter(dnnl::impl::cpu::x64::jit_generator* host, dnnl::impl::cpu::x64::cpu_isa_t host_isa, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32); + jit_bitwise_and_emitter(dnnl::impl::cpu::x64::jit_generator* host, dnnl::impl::cpu::x64::cpu_isa_t host_isa, const std::shared_ptr& n, + InferenceEngine::Precision exec_prc = InferenceEngine::Precision::FP32); + + size_t get_inputs_num() const override; + static std::set> get_supported_precisions(const std::shared_ptr& node = nullptr); + +private: + void emit_impl(const std::vector& in_vec_idxs, const std::vector& out_vec_idxs) const override; + + template + void emit_isa(const std::vector& in_vec_idxs, const std::vector& out_vec_idxs) const; + + void register_table_entries() override; + size_t aux_vecs_count() const override; +}; + } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/eltwise.cpp b/src/plugins/intel_cpu/src/nodes/eltwise.cpp index 852acc7487d318..182a9e633a8320 100644 --- a/src/plugins/intel_cpu/src/nodes/eltwise.cpp +++ b/src/plugins/intel_cpu/src/nodes/eltwise.cpp @@ -36,6 +36,7 @@ #include "ngraph/ngraph.hpp" #include +#include #include "transformations/cpu_opset/common/op/power_static.hpp" #include "transformations/cpu_opset/common/op/leaky_relu.hpp" #include "transformations/cpu_opset/common/op/swish_cpu.hpp" @@ -245,6 +246,10 @@ std::set> eltwise_precision_helper::get_supported_pre OV_CASE(Algorithm::EltwiseIsInf, jit_is_inf_emitter), OV_CASE(Algorithm::EltwiseIsNaN, jit_is_nan_emitter), OV_CASE(Algorithm::EltwiseSelect, jit_select_emitter)); + //OV_CASE(Algorithm::EltwiseBitwiseAnd, jit_bitwise_and_emitter), + //OV_CASE(Algorithm::EltwiseBitwiseOr, jit_bitwise_and_emitter), + //OV_CASE(Algorithm::EltwiseBitwiseNot, jit_bitwise_and_emitter), + //OV_CASE(Algorithm::EltwiseBitwiseXor, jit_bitwise_and_emitter)); if (precisions.empty()) IE_THROW() << "Unsupported operation type for Eltwise emitter"; @@ -619,6 +624,10 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener OV_CASE(Algorithm::EltwiseIsInf, jit_is_inf_emitter), OV_CASE(Algorithm::EltwiseIsNaN, jit_is_nan_emitter), OV_CASE(Algorithm::EltwiseSelect, jit_select_emitter)); + //OV_CASE(Algorithm::EltwiseBitwiseAnd, jit_bitwise_and_emitter), + //OV_CASE(Algorithm::EltwiseBitwiseOr, jit_bitwise_and_emitter), + //OV_CASE(Algorithm::EltwiseBitwiseNot, jit_bitwise_and_emitter), + //OV_CASE(Algorithm::EltwiseBitwiseXor, jit_bitwise_and_emitter)); if (!ctx.emitter) IE_THROW() << "Unsupported operation type for Eltwise emitter"; @@ -717,7 +726,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener uni_vpmovzxbd(vmm_src, op); break; default: - assert(!"unknown src_prc"); + IE_THROW() << "unknown src_prc: " << src_prc; } switch (dst_prc) { @@ -730,7 +739,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener uni_vcvtps2dq(vmm_src, vmm_src); break; default: - assert(!"unknown dst_prc"); + IE_THROW() << "unknown dst_prc: " << dst_prc; } } } @@ -765,7 +774,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener uni_vmovq(xmm_src, reg_tmp_64); break; default: - assert(!"unknown src_prc"); + IE_THROW() << "unknown src_prc: " << src_prc; } switch (dst_prc) { @@ -778,7 +787,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener uni_vcvtps2dq(xmm_src, xmm_src); break; default: - assert(!"unknown dst_prc"); + IE_THROW() << "unknown dst_prc: " << dst_prc; } } @@ -796,7 +805,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener uni_vcvtdq2ps(vmm_dst, vmm_dst); break; default: - assert(!"unknown src_prc"); + IE_THROW() << "unknown src_prc: " << src_prc; } switch (dst_prc) { @@ -868,7 +877,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener } break; default: - assert(!"unknown dst_prc"); + IE_THROW() << "unknown dst_prc: " << dst_prc; } } @@ -883,7 +892,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener uni_vcvtdq2ps(xmm_dst, xmm_dst); break; default: - assert(!"unknown src_prc"); + IE_THROW() << "unknown src_prc: " << src_prc; } switch (dst_prc) { @@ -923,7 +932,7 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener mov(op, reg_tmp_8); break; default: - assert(!"unknown dst_prc"); + IE_THROW() << "unknown dst_prc: " << dst_prc; } } }; @@ -1160,6 +1169,18 @@ const std::map& Eltwise::g {ngraph::op::v0::Log::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { node.algorithm = Algorithm::EltwiseLog; }}, + {op::v13::BitwiseAnd::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { + node.algorithm = Algorithm::EltwiseBitwiseAnd; + }}, + {op::v13::BitwiseNot::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { + node.algorithm = Algorithm::EltwiseBitwiseNot; + }}, + {op::v13::BitwiseOr::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { + node.algorithm = Algorithm::EltwiseBitwiseOr; + }}, + {op::v13::BitwiseXor::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { + node.algorithm = Algorithm::EltwiseBitwiseXor; + }}, }; return initializers; } @@ -1735,6 +1756,10 @@ class EltwiseRefExecutor : public Eltwise::IEltwiseExecutor { break; case Algorithm::EltwiseIsNaN: *dst_ptr_f = std::isnan(src_f[0]); break; case Algorithm::EltwiseSelect: *dst_ptr_f = src_f[0] ? src_f[1] : src_f[2]; break; + case Algorithm::EltwiseBitwiseAnd: *dst_ptr_f = static_cast(src_f[0]) & static_cast(src_f[1]); break; + //case Algorithm::EltwiseBitwiseNot: *dst_ptr_f = ~src_f[0]; break; + //case Algorithm::EltwiseBitwiseOr: *dst_ptr_f = src_f[0] | src_f[1]; break; + //case Algorithm::EltwiseBitwiseXor: *dst_ptr_f = src_f[0] ^ src_f[1]; break; default: IE_THROW() << "Unsupported operation type for Eltwise executor"; } } @@ -1885,6 +1910,12 @@ size_t Eltwise::getOpInputsNum() const { case Algorithm::EltwiseMulAdd: case Algorithm::EltwiseSelect: return 3; + case Algorithm::EltwiseBitwiseAnd: + case Algorithm::EltwiseBitwiseOr: + case Algorithm::EltwiseBitwiseXor: + return 2; + case Algorithm::EltwiseBitwiseNot: + return 1; default: IE_THROW() << "Unsupported operation for Eltwise node with name `" << getName() << "`."; } } @@ -1926,7 +1957,8 @@ void Eltwise::initSupportedPrimitiveDescriptors() { // if dim rank is greater than the maximum possible, we should use the reference execution bool canUseOptimizedImpl = mayiuse(x64::sse41) && getInputShapeAtPort(0).getRank() <= MAX_ELTWISE_DIM_RANK; // TODO: Add EltwiseLog algorithm support for JIT implementation - canUseOptimizedImpl &= !one_of(getAlgorithm(), Algorithm::EltwiseLog); + canUseOptimizedImpl &= !one_of(getAlgorithm(), Algorithm::EltwiseLog) && !one_of(getAlgorithm(), Algorithm::EltwiseBitwiseAnd); + bool canUseOptimizedShapeAgnosticImpl = isDynamicNode() && canUseOptimizedImpl; if (!canUseOptimizedImpl && !fusedWith.empty()) { diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/eltwise.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/eltwise.cpp index bf61d07e289d25..780a9a2d0325e3 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/eltwise.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/eltwise.cpp @@ -219,5 +219,33 @@ const auto params_5D_dyn_param = ::testing::Combine( INSTANTIATE_TEST_SUITE_P(smoke_CompareWithRefs_5D_MemOrder_dyn_param, EltwiseLayerCPUTest, params_5D_dyn_param, EltwiseLayerCPUTest::getTestCaseName); +static const std::vector> bitwise_in_shapes_4D = { + {{1, 3, 4, 4}, {1, 3, 4, 4}}, + {{1, 3, 4, 4}, {1, 3, 1, 1}}, +}; + +// TODO: debug: for development only +const auto params_4D_bitwise = ::testing::Combine( + ::testing::Combine( + ::testing::ValuesIn(static_shapes_to_test_representation(bitwise_in_shapes_4D)), + ::testing::ValuesIn({ + ngraph::helpers::EltwiseTypes::BITWISE_AND, + ngraph::helpers::EltwiseTypes::BITWISE_NOT, + ngraph::helpers::EltwiseTypes::BITWISE_OR, + ngraph::helpers::EltwiseTypes::BITWISE_XOR + }), + ::testing::ValuesIn(secondaryInputTypes()), + ::testing::ValuesIn({ ov::test::utils::OpType::VECTOR }), + ::testing::ValuesIn({ ElementType::boolean }), + ::testing::Values(ov::element::boolean), + ::testing::Values(ov::element::boolean), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::ValuesIn(additional_config())), + ::testing::ValuesIn(filterCPUSpecificParams(cpuParams_4D())), + ::testing::Values(emptyFusingSpec), + ::testing::Values(false)); + +INSTANTIATE_TEST_SUITE_P(smoke_CompareWithRefs_4D_Bitwise, EltwiseLayerCPUTest, params_4D_bitwise, EltwiseLayerCPUTest::getTestCaseName); + } // namespace Eltwise } // namespace CPULayerTestsDefinitions diff --git a/src/tests/ov_helpers/ov_models/src/eltwise.cpp b/src/tests/ov_helpers/ov_models/src/eltwise.cpp index 4932332e0773fb..25cfe13bc1a436 100644 --- a/src/tests/ov_helpers/ov_models/src/eltwise.cpp +++ b/src/tests/ov_helpers/ov_models/src/eltwise.cpp @@ -7,6 +7,8 @@ #include "common_test_utils/test_enums.hpp" #include "ov_models/utils/ov_helpers.hpp" +#include "openvino/opsets/opset13.hpp" + namespace ngraph { namespace builder { @@ -32,6 +34,14 @@ std::shared_ptr makeEltwise(const ov::Output& in0, return std::make_shared(in0, in1); case ov::test::utils::EltwiseTypes::ERF: return std::make_shared(in0); + case ngraph::helpers::EltwiseTypes::BITWISE_AND: + return std::make_shared(in0, in1); + case ngraph::helpers::EltwiseTypes::BITWISE_NOT: + return std::make_shared(in0); + case ngraph::helpers::EltwiseTypes::BITWISE_OR: + return std::make_shared(in0, in1); + case ngraph::helpers::EltwiseTypes::BITWISE_XOR: + return std::make_shared(in0, in1); default: { throw std::runtime_error("Incorrect type of Eltwise operation"); } diff --git a/src/tests/test_utils/common_test_utils/include/common_test_utils/test_enums.hpp b/src/tests/test_utils/common_test_utils/include/common_test_utils/test_enums.hpp index 6e73dd07a5adac..5c93d211cac30b 100644 --- a/src/tests/test_utils/common_test_utils/include/common_test_utils/test_enums.hpp +++ b/src/tests/test_utils/common_test_utils/include/common_test_utils/test_enums.hpp @@ -56,7 +56,11 @@ enum EltwiseTypes { POWER, FLOOR_MOD, MOD, - ERF + ERF, + BITWISE_AND, + BITWISE_NOT, + BITWISE_OR, + BITWISE_XOR }; enum SqueezeOpType { diff --git a/src/tests/test_utils/common_test_utils/src/test_enums.cpp b/src/tests/test_utils/common_test_utils/src/test_enums.cpp index 8bb1cff3ce77dc..e67122d9b8af4f 100644 --- a/src/tests/test_utils/common_test_utils/src/test_enums.cpp +++ b/src/tests/test_utils/common_test_utils/src/test_enums.cpp @@ -70,6 +70,18 @@ std::ostream& operator<<(std::ostream& os, const ov::test::utils::EltwiseTypes t case ov::test::utils::EltwiseTypes::ERF: os << "Erf"; break; + case ov::test::utils::EltwiseTypes::BITWISE_AND: + os << "BitwiseAnd"; + break; + case ov::test::utils::EltwiseTypes::BITWISE_NOT: + os << "BitwiseNot"; + break; + case ov::test::utils::EltwiseTypes::BITWISE_OR: + os << "BitwiseOr"; + break; + case ov::test::utils::EltwiseTypes::BITWISE_XOR: + os << "BitwiseXor"; + break; default: throw std::runtime_error("NOT_SUPPORTED_OP_TYPE"); }