diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp index 5ada89073046e5..1dbd769444b1aa 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp @@ -264,6 +264,9 @@ REGISTER_FACTORY(v12, ScatterElementsUpdate); // ------------------------------ Supported v13 ops ----------------------------- // REGISTER_FACTORY(v13, Multinomial); REGISTER_FACTORY(v13, ScaledDotProductAttention); +REGISTER_FACTORY(v13, BitwiseAnd); +REGISTER_FACTORY(v13, BitwiseOr); +REGISTER_FACTORY(v13, BitwiseXor); // ------------------------------ Supported v15 ops ----------------------------- // REGISTER_FACTORY(v15, ROIAlignRotated); diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp index 7913435324e89d..4083c1e9c8adf5 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp @@ -58,6 +58,12 @@ enum class eltwise_mode : int32_t { right_shift, /// @brief Eltwise bitwise left shift. left_shift, + /// @brief Eltwise bitwise and. + bitwise_and, + /// @brief Eltwise bitwise or. + bitwise_or, + /// @brief Eltwise bitwise xor. + bitwise_xor }; /// @brief Performs elementwise operations (sum, subtract, max or product) on two input primitives diff --git a/src/plugins/intel_gpu/src/graph/eltwise.cpp b/src/plugins/intel_gpu/src/graph/eltwise.cpp index 0791a44014a079..a370e8ba260f8b 100644 --- a/src/plugins/intel_gpu/src/graph/eltwise.cpp +++ b/src/plugins/intel_gpu/src/graph/eltwise.cpp @@ -81,7 +81,10 @@ layout eltwise_inst::calc_output_layout(eltwise_node const& node, kernel_impl_pa eltwise_mode::logic_or, eltwise_mode::logic_xor, eltwise_mode::right_shift, - eltwise_mode::left_shift}; + eltwise_mode::left_shift, + eltwise_mode::bitwise_and, + eltwise_mode::bitwise_or, + eltwise_mode::bitwise_xor}; if (std::find(eltwise_int_modes.begin(), eltwise_int_modes.end(), mode) == eltwise_int_modes.end()) CLDNN_ERROR_MESSAGE(desc->id, "Requested eltwise mode is not supported for integer types."); } @@ -181,7 +184,10 @@ std::vector eltwise_inst::calc_output_layouts(eltwise_node const& /*node eltwise_mode::logic_or, eltwise_mode::logic_xor, eltwise_mode::right_shift, - eltwise_mode::left_shift}; + eltwise_mode::left_shift, + eltwise_mode::bitwise_and, + eltwise_mode::bitwise_or, + eltwise_mode::bitwise_xor}; OPENVINO_ASSERT((std::find(eltwise_int_modes.begin(), eltwise_int_modes.end(), mode) != eltwise_int_modes.end()), desc->id + "Requested eltwise mode is not supported for integer types."); @@ -315,6 +321,15 @@ std::string eltwise_inst::to_string(eltwise_node const& node) { case eltwise_mode::left_shift: str_mode = "left_shift"; break; + case eltwise_mode::bitwise_and: + str_mode = "bitwise_and"; + break; + case eltwise_mode::bitwise_or: + str_mode = "bitwise_or"; + break; + case eltwise_mode::bitwise_xor: + str_mode = "bitwise_xor"; + break; default: str_mode = "not supported mode"; break; @@ -440,6 +455,9 @@ void eltwise_inst::check_inputs_count(eltwise_node const& node) { case eltwise_mode::floor_mod: case eltwise_mode::right_shift: case eltwise_mode::left_shift: + case eltwise_mode::bitwise_and: + case eltwise_mode::bitwise_or: + case eltwise_mode::bitwise_xor: OPENVINO_ASSERT(inputs_number == 2, "Node id: ", node.id(), ". Invalid eltwise inputs number (should be equal to 2). Actual: ", inputs_number); break; diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp index 8b95cfc626fa44..1b18b78b041fd8 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp @@ -2,35 +2,37 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "register.hpp" #include "eltwise_inst.h" #include "impls/registry/implementation_map.hpp" - #include "openvino/op/add.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/maximum.hpp" -#include "openvino/op/minimum.hpp" -#include "openvino/op/subtract.hpp" +#include "openvino/op/bitwise_and.hpp" +#include "openvino/op/bitwise_left_shift.hpp" +#include "openvino/op/bitwise_or.hpp" +#include "openvino/op/bitwise_right_shift.hpp" +#include "openvino/op/bitwise_xor.hpp" #include "openvino/op/divide.hpp" -#include "openvino/op/squared_difference.hpp" #include "openvino/op/equal.hpp" -#include "openvino/op/not_equal.hpp" -#include "openvino/op/less.hpp" -#include "openvino/op/less_eq.hpp" +#include "openvino/op/floor_mod.hpp" #include "openvino/op/greater.hpp" #include "openvino/op/greater_eq.hpp" +#include "openvino/op/is_finite.hpp" +#include "openvino/op/is_inf.hpp" +#include "openvino/op/is_nan.hpp" +#include "openvino/op/less.hpp" +#include "openvino/op/less_eq.hpp" #include "openvino/op/logical_and.hpp" #include "openvino/op/logical_or.hpp" #include "openvino/op/logical_xor.hpp" -#include "openvino/op/xor.hpp" -#include "openvino/op/power.hpp" -#include "openvino/op/floor_mod.hpp" +#include "openvino/op/maximum.hpp" +#include "openvino/op/minimum.hpp" #include "openvino/op/mod.hpp" -#include "openvino/op/is_finite.hpp" -#include "openvino/op/is_inf.hpp" -#include "openvino/op/is_nan.hpp" -#include "openvino/op/bitwise_right_shift.hpp" -#include "openvino/op/bitwise_left_shift.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/not_equal.hpp" +#include "openvino/op/power.hpp" +#include "openvino/op/squared_difference.hpp" +#include "openvino/op/subtract.hpp" +#include "openvino/op/xor.hpp" +#include "register.hpp" namespace cldnn { namespace cpu { @@ -174,6 +176,15 @@ struct eltwise_impl : public typed_primitive_impl { case eltwise_mode::left_shift: op = std::make_shared(); break; + case eltwise_mode::bitwise_and: + op = std::make_shared(); + break; + case eltwise_mode::bitwise_or: + op = std::make_shared(); + break; + case eltwise_mode::bitwise_xor: + op = std::make_shared(); + break; default: OPENVINO_THROW("[GPU] Couldn't create eltwise operation: unsupported eltwise operation (", static_cast(mode), ")"); } diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h index 59a7e8e9627b14..3ddb5bf8793c29 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h @@ -204,6 +204,12 @@ inline kernel_selector::eltwise_mode convert_to_eltwise_mode(eltwise_mode mode) return kernel_selector::eltwise_mode::RIGHT_SHIFT; case eltwise_mode::left_shift: return kernel_selector::eltwise_mode::LEFT_SHIFT; + case eltwise_mode::bitwise_and: + return kernel_selector::eltwise_mode::BITWISE_AND; + case eltwise_mode::bitwise_or: + return kernel_selector::eltwise_mode::BITWISE_OR; + case eltwise_mode::bitwise_xor: + return kernel_selector::eltwise_mode::BITWISE_XOR; default: OPENVINO_ASSERT(false, "Unsupported eltwise mode!"); return kernel_selector::eltwise_mode::ADD; diff --git a/src/plugins/intel_gpu/src/kernel_selector/common_types.h b/src/plugins/intel_gpu/src/kernel_selector/common_types.h index 73717d76933205..bc9cc9f5b8da07 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/common_types.h +++ b/src/plugins/intel_gpu/src/kernel_selector/common_types.h @@ -308,6 +308,9 @@ enum class EltwiseMode { IS_NAN, RIGHT_SHIFT, LEFT_SHIFT, + BITWISE_AND, + BITWISE_OR, + BITWISE_XOR }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/eltwise/eltwise_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/eltwise/eltwise_kernel_base.cpp index ef46ede8a9f7e9..70aa2a87c3eaf9 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/eltwise/eltwise_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/eltwise/eltwise_kernel_base.cpp @@ -51,6 +51,9 @@ uint32_t GetNumberOfInputs(EltwiseMode m) { case EltwiseMode::FLOOR_MOD: case EltwiseMode::RIGHT_SHIFT: case EltwiseMode::LEFT_SHIFT: + case EltwiseMode::BITWISE_AND: + case EltwiseMode::BITWISE_OR: + case EltwiseMode::BITWISE_XOR: return 2; case EltwiseMode::SQRT: case EltwiseMode::RSQRT: @@ -79,12 +82,17 @@ ParamsKey eltwise_params::GetParamsKey() const { return k; } +static bool IsBitwiseMode(EltwiseMode mode) { + return mode == EltwiseMode::BITWISE_AND || mode == EltwiseMode::LEFT_SHIFT || mode == EltwiseMode::RIGHT_SHIFT || + mode == EltwiseMode::BITWISE_OR || mode == EltwiseMode::BITWISE_XOR; +} + Datatype EltwiseKernelBase::GetAccumulatorType(const eltwise_params ¶ms) const { // NOTE: Workaround for not promoting shift operations. Not sure what should happen // if shift op is just one operation of other elementwise operations. My guess is that is should be promoted as // well, but in reality more robust solution will be needed or (better) - assumption that types are not promoted. So // probably this is a temporary solution. - if (params.operations[0].mode == EltwiseMode::RIGHT_SHIFT || params.operations[0].mode == EltwiseMode::LEFT_SHIFT) { + if (IsBitwiseMode(params.operations[0].mode)) { return params.inputs[0].GetDType(); } @@ -329,6 +337,15 @@ JitConstants EltwiseKernelBase::GetOperationsJitConstants(const eltwise_params& case EltwiseMode::LEFT_SHIFT: op += "(" + input0_str + " << " + input1_str + ")"; break; + case EltwiseMode::BITWISE_AND: + op += "(" + input0_str + " & " + input1_str + ")"; + break; + case EltwiseMode::BITWISE_OR: + op += "(" + input0_str + " | " + input1_str + ")"; + break; + case EltwiseMode::BITWISE_XOR: + op += "(" + input0_str + " ^ " + input1_str + ")"; + break; default: break; } diff --git a/src/plugins/intel_gpu/src/plugin/ops/eltwise.cpp b/src/plugins/intel_gpu/src/plugin/ops/eltwise.cpp index 2866e9151e214d..ad0eb26c6e1cc0 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/eltwise.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/eltwise.cpp @@ -2,40 +2,42 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "intel_gpu/plugin/program_builder.hpp" -#include "intel_gpu/plugin/common_utils.hpp" -#include "transformations/utils/utils.hpp" +#include "intel_gpu/primitives/eltwise.hpp" +#include "intel_gpu/plugin/common_utils.hpp" +#include "intel_gpu/plugin/program_builder.hpp" +#include "intel_gpu/primitives/activation.hpp" +#include "intel_gpu/primitives/reorder.hpp" +#include "intel_gpu/primitives/reshape.hpp" #include "openvino/op/add.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/maximum.hpp" -#include "openvino/op/minimum.hpp" -#include "openvino/op/subtract.hpp" +#include "openvino/op/bitwise_and.hpp" +#include "openvino/op/bitwise_or.hpp" +#include "openvino/op/bitwise_xor.hpp" +#include "openvino/op/bitwise_left_shift.hpp" +#include "openvino/op/bitwise_right_shift.hpp" #include "openvino/op/divide.hpp" -#include "openvino/op/squared_difference.hpp" #include "openvino/op/equal.hpp" -#include "openvino/op/not_equal.hpp" +#include "openvino/op/floor_mod.hpp" +#include "openvino/op/greater.hpp" +#include "openvino/op/greater_eq.hpp" #include "openvino/op/is_finite.hpp" #include "openvino/op/is_inf.hpp" #include "openvino/op/is_nan.hpp" #include "openvino/op/less.hpp" #include "openvino/op/less_eq.hpp" -#include "openvino/op/mod.hpp" -#include "openvino/op/greater.hpp" -#include "openvino/op/greater_eq.hpp" #include "openvino/op/logical_and.hpp" #include "openvino/op/logical_or.hpp" #include "openvino/op/logical_xor.hpp" -#include "openvino/op/xor.hpp" +#include "openvino/op/maximum.hpp" +#include "openvino/op/minimum.hpp" +#include "openvino/op/mod.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/not_equal.hpp" #include "openvino/op/power.hpp" -#include "openvino/op/floor_mod.hpp" -#include "openvino/op/bitwise_right_shift.hpp" -#include "openvino/op/bitwise_left_shift.hpp" - -#include "intel_gpu/primitives/activation.hpp" -#include "intel_gpu/primitives/eltwise.hpp" -#include "intel_gpu/primitives/reorder.hpp" -#include "intel_gpu/primitives/reshape.hpp" +#include "openvino/op/squared_difference.hpp" +#include "openvino/op/subtract.hpp" +#include "openvino/op/xor.hpp" +#include "transformations/utils/utils.hpp" namespace ov { namespace intel_gpu { @@ -208,6 +210,18 @@ static void CreateBitwiseLeftShiftOp(ProgramBuilder& p, const std::shared_ptr& op) { + CreateElementwiseOp(p, op, cldnn::eltwise_mode::bitwise_and); +} + +static void CreateBitwiseOrOp(ProgramBuilder& p, const std::shared_ptr& op) { + CreateElementwiseOp(p, op, cldnn::eltwise_mode::bitwise_or); +} + +static void CreateBitwiseXorOp(ProgramBuilder& p, const std::shared_ptr& op) { + CreateElementwiseOp(p, op, cldnn::eltwise_mode::bitwise_xor); +} + REGISTER_FACTORY_IMPL(v1, Add); REGISTER_FACTORY_IMPL(v1, Multiply); REGISTER_FACTORY_IMPL(v1, Maximum); @@ -230,6 +244,9 @@ REGISTER_FACTORY_IMPL(v1, Mod); REGISTER_FACTORY_IMPL(v10, IsFinite); REGISTER_FACTORY_IMPL(v10, IsInf); REGISTER_FACTORY_IMPL(v10, IsNaN); +REGISTER_FACTORY_IMPL(v13, BitwiseAnd); +REGISTER_FACTORY_IMPL(v13, BitwiseOr); +REGISTER_FACTORY_IMPL(v13, BitwiseXor); REGISTER_FACTORY_IMPL(v15, BitwiseRightShift); REGISTER_FACTORY_IMPL(v15, BitwiseLeftShift); diff --git a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/single_layer_tests/eltwise.cpp b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/single_layer_tests/eltwise.cpp index 3109daecb439bf..85ba33753a7407 100644 --- a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/single_layer_tests/eltwise.cpp +++ b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/single_layer_tests/eltwise.cpp @@ -67,7 +67,8 @@ std::vector eltwiseOpTypes = { }; std::vector smoke_intOnly_eltwiseOpTypes = { - EltwiseTypes::RIGHT_SHIFT + EltwiseTypes::RIGHT_SHIFT, + EltwiseTypes::BITWISE_AND }; std::vector intOnly_netPrecisions = { diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/eltwise_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/eltwise_gpu_test.cpp index 118691bb29c9ff..5900edff6bec11 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/eltwise_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/eltwise_gpu_test.cpp @@ -193,6 +193,12 @@ TOut eltwise_int_execute(cldnn::eltwise_mode mode, T x, T y) { return x >> y; case eltwise_mode::left_shift: return x << y; + case eltwise_mode::bitwise_and: + return x & y; + case eltwise_mode::bitwise_or: + return x | y; + case eltwise_mode::bitwise_xor: + return x ^ y; default: return (TOut)0; } @@ -353,6 +359,37 @@ void run_eltwise_int_shift_generic_test(cldnn::eltwise_mode mode) { ELTWISE_INT_TEST_CASES(uint32_t); ELTWISE_INT_TEST_CASES(int64_t); +#undef ELTWISE_INT_TEST_CASES +} + +void run_eltwise_int_bitwise_generic_test(cldnn::eltwise_mode mode) { + cldnn::format test_inputs_fmt = cldnn::format::bfyx; + const int dim_size = 227; + +#define ELTWISE_INT_TEST_CASES(type) \ + generic_eltwise_int_test(test_inputs_fmt, \ + 1, \ + 1, \ + dim_size, \ + dim_size, \ + mode, \ + 0, \ + 0, \ + 0, \ + 0, \ + 0, \ + static_cast(std::numeric_limits::max()), \ + 0, \ + static_cast(std::numeric_limits::max())); + + ELTWISE_INT_TEST_CASES(int8_t); + ELTWISE_INT_TEST_CASES(uint8_t); + ELTWISE_INT_TEST_CASES(int16_t); + ELTWISE_INT_TEST_CASES(uint16_t); + ELTWISE_INT_TEST_CASES(int32_t); + ELTWISE_INT_TEST_CASES(uint32_t); + ELTWISE_INT_TEST_CASES(int64_t); + #undef ELTWISE_INT_TEST_CASES } } // namespace @@ -3955,6 +3992,18 @@ TEST(eltwise_gpu, eltwise_left_shift) { run_eltwise_int_shift_generic_test(cldnn::eltwise_mode::left_shift); } +TEST(eltwise_gpu, eltwise_bitwise_and) { + run_eltwise_int_bitwise_generic_test(cldnn::eltwise_mode::bitwise_and); +} + +TEST(eltwise_gpu, eltwise_bitwise_or) { + run_eltwise_int_bitwise_generic_test(cldnn::eltwise_mode::bitwise_or); +} + +TEST(eltwise_gpu, eltwise_bitwise_xor) { + run_eltwise_int_bitwise_generic_test(cldnn::eltwise_mode::bitwise_xor); +} + TEST(eltwise_gpu, eltwise_div) { run_eltwise_generic_test(cldnn::eltwise_mode::div); }