From f523c7a9e431f7e1d0af39c665428a14ad5806af Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Fri, 27 Oct 2023 14:09:57 +0400 Subject: [PATCH 1/9] RPE operation and fusing transformation --- .../ov_ops/rotary_positional_embeddings.hpp | 46 +++++++++ .../fuse_rotary_positional_embeddings.hpp | 25 +++++ .../ov_ops/rotary_positional_embeddings.cpp | 32 +++++++ .../fuse_rotary_positional_embeddings.cpp | 94 +++++++++++++++++++ .../fuse_rotary_positional_embeddings.cpp | 91 ++++++++++++++++++ .../transformation_pipeline.cpp | 3 + 6 files changed, 291 insertions(+) create mode 100644 src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp create mode 100644 src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp create mode 100644 src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp create mode 100644 src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp create mode 100644 src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp diff --git a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp new file mode 100644 index 00000000000000..2a03637f6f7383 --- /dev/null +++ b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp @@ -0,0 +1,46 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "openvino/op/op.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace op { +namespace internal { + +/// +/// \brief Rotary Positional Embeddings operation +/// Internal operation which may change in the future +/// \ingroup ov_ops_cpp_api +class TRANSFORMATIONS_API RPE : public ov::op::Op { +public: + OPENVINO_OP("RPE", "ie_internal_opset", op::Op); + + RPE() = default; + RPE(const Output& data, const Output& sin, const Output& cos, const int64_t& axis); + + bool visit_attributes(AttributeVisitor& visitor) override; + void validate_and_infer_types() override; + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + + int64_t get_axis() const { + return m_axis; + }; + void set_axis(const int64_t& axis) { + m_axis = axis; + }; + +private: + int64_t m_axis{}; +}; + +} // namespace internal +} // namespace op +} // namespace ov diff --git a/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp b/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp new file mode 100644 index 00000000000000..746b0e41ec58ee --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp @@ -0,0 +1,25 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +namespace ov { +namespace pass { +class TRANSFORMATIONS_API RPE_Fusion; +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief Fuses special sub-graph into an internal Rotary Positional Embedding operation + */ +class ov::pass::RPE_Fusion : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("RPE_Fusion", "0"); + RPE_Fusion(); +}; diff --git a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp new file mode 100644 index 00000000000000..accd8a038e8baa --- /dev/null +++ b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ov_ops/rotary_positional_embeddings.hpp" + +#include "itt.hpp" + +using namespace std; +using namespace ov::op::internal; + +RPE::RPE(const Output& data, const Output& sin, const Output& cos, const int64_t& axis) + : Op({data, sin, cos}), + m_axis{axis} { + constructor_validate_and_infer_types(); +} + +void RPE::validate_and_infer_types() { + INTERNAL_OP_SCOPE(internal_RoPE_validate_and_infer_types); + set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); +} + +bool RPE::visit_attributes(ov::AttributeVisitor& visitor) { + INTERNAL_OP_SCOPE(internal_RoPE_visit_attributes); + visitor.on_attribute("axis", m_axis); + return true; +} + +shared_ptr RPE::clone_with_new_inputs(const ov::OutputVector& new_args) const { + INTERNAL_OP_SCOPE(internal_RoPE_clone_with_new_inputs); + return make_shared(new_args.at(0), new_args.at(1), new_args.at(2), m_axis); +} diff --git a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp new file mode 100644 index 00000000000000..e3d6b3f7a9d5e6 --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -0,0 +1,94 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itt.hpp" +#include "transformations/utils/utils.hpp" + +ov::pass::RPE_Fusion::RPE_Fusion() { + MATCHER_SCOPE(RPE_Fusion); + + auto sin = pattern::any_input(); + auto cos = pattern::any_input(); + + // FIXME: should be a single node match + auto source_1 = pattern::any_input(); + auto source = pattern::any_input(); + // BEGIN: rotate_half + + // Variadic Split into two equal parts + auto axis = pattern::any_input(); + auto split_length = INT_CONSTANT_WITH_PREDICATE(value.size() == 2 && value[0] == value[1]); + auto vsplit = pattern::wrap_type({source, axis, split_length}); + vsplit->set_output_size(2); + + // Negate + auto minus_1 = FLOAT_CONSTANT_WITH_PREDICATE(value.size() == 1 && value[0] == -1); + auto neg = pattern::wrap_type({vsplit->output(1), minus_1}); + + // Concat two splitted parts in the opposite order, first of them is negated + auto concat = pattern::wrap_type({neg, vsplit->output(0)}); // make sure axis eq to vsplit eq -1 + + // END: rotate half + + auto mul_sin = pattern::wrap_type({concat, sin}); + auto mul_cos = pattern::wrap_type({source_1, cos}); + auto add = pattern::wrap_type({mul_cos, mul_sin}); + + ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { + auto value_map = m.get_pattern_value_map(); + + auto actual_source = value_map.at(vsplit).get_node_shared_ptr()->input_value(0); + auto potential_source = value_map.at(mul_cos).get_node_shared_ptr()->input_value(0); + auto cos_output = value_map.at(mul_cos).get_node_shared_ptr()->input_value(1); + + if (actual_source != potential_source && actual_source != cos_output) + return false; // flawed match + if (actual_source == potential_source && actual_source == cos_output) + return false; // flawed match + if (actual_source != potential_source && actual_source == cos_output) + cos_output = potential_source; + + auto input = value_map.at(source); + auto concat_node = ov::as_type_ptr(value_map.at(concat).get_node_shared_ptr()); + if (!concat_node) + return false; + OPENVINO_SUPPRESS_DEPRECATED_START + auto split_axis_node = ov::get_constant_from_source(value_map.at(axis)); + OPENVINO_SUPPRESS_DEPRECATED_END + if (!split_axis_node) + return false; + auto value = split_axis_node->cast_vector(); + if (value.size() != 1) + return false; + auto concat_axis = concat_node->get_concatenation_axis(); + auto split_axis = value[0]; + if (concat_axis != split_axis) { + if (input.get_partial_shape().is_static()) { + auto rank = input.get_partial_shape().rank().get_length(); + concat_axis = concat_axis < 0 ? concat_axis + rank : concat_axis; + split_axis = split_axis < 0 ? split_axis + rank : split_axis; + } + if (concat_axis != split_axis) + return false; + } + auto rpe = + std::make_shared(input, value_map.at(sin), cos_output, concat_node->get_axis()); + ov::replace_output_update_name(value_map.at(add), rpe->output(0)); + return true; + }; + auto m = std::make_shared(add, matcher_name); + register_matcher(m, matcher_pass_callback); +} diff --git a/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp new file mode 100644 index 00000000000000..584b46c7fd222c --- /dev/null +++ b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include +#include +#include + +#include "common_test_utils/ov_test_utils.hpp" + +using namespace std; +using namespace ov; +using namespace ov::op; + +void name_node_and_output(const shared_ptr& op, const std::string& name) { + op->set_friendly_name(name); + op->output(0).set_names({name}); +} + +TEST_F(TransformationTestsF, FuseRPE) { + { + auto data = make_shared(element::f32, PartialShape::dynamic()); + name_node_and_output(data, "source"); + auto sin = make_shared(element::f32, PartialShape::dynamic()); + name_node_and_output(sin, "sin"); + auto cos = make_shared(element::f32, PartialShape::dynamic()); + name_node_and_output(cos, "cos"); + auto axis = v0::Constant::create(element::i64, {}, {-1}); + auto split_lengths = v0::Constant::create(element::i64, {2}, {10, 10}); + auto split = make_shared(data, axis, split_lengths); + + auto minus_one = v0::Constant::create(element::f32, {}, {-1}); + auto negate = make_shared(split->output(1), minus_one); + + auto concat = make_shared(OutputVector{negate, split->output(0)}, -1); + + auto mul_sin = make_shared(concat, sin); + auto mul_cos = make_shared(data, cos); + auto add = make_shared(mul_cos, mul_sin); + name_node_and_output(add, "rpe"); + + model = std::make_shared(NodeVector{add}, ParameterVector{data, sin, cos}); + + manager.register_pass(); + } + { + auto data = make_shared(element::f32, PartialShape::dynamic()); + name_node_and_output(data, "source"); + auto sin = make_shared(element::f32, PartialShape::dynamic()); + name_node_and_output(sin, "sin"); + auto cos = make_shared(element::f32, PartialShape::dynamic()); + name_node_and_output(cos, "cos"); + auto rpe = make_shared(data, sin, cos, -1); + name_node_and_output(rpe, "rpe"); + model_ref = std::make_shared(NodeVector{rpe}, ParameterVector{data, sin, cos}); + } + comparator.enable(FunctionsComparator::CmpValues::NAMES); +} + +TEST_F(TransformationTestsF, FuseRPESorcesAreMultiOutputed) { + { + auto data_ = make_shared(element::f32, PartialShape::dynamic()); + auto sin_ = make_shared(element::f32, PartialShape::dynamic()); + auto cos_ = make_shared(element::f32, PartialShape::dynamic()); + + auto data = make_shared(data_, v0::Constant::create(element::i64, {}, {-1}), 2); + auto sin = make_shared(sin_, v0::Constant::create(element::i64, {}, {-1}), 2)->output(1); + auto cos = make_shared(cos_, v0::Constant::create(element::i64, {}, {-1}), 2)->output(1); + + auto axis = v0::Constant::create(element::i64, {}, {-1}); + auto split_lengths = v0::Constant::create(element::i64, {2}, {10, 10}); + auto split = make_shared(data->output(0), axis, split_lengths); + + auto minus_one = v0::Constant::create(element::f32, {}, {-1}); + auto negate = make_shared(split->output(1), minus_one); + + auto concat = make_shared(OutputVector{negate, split->output(0)}, -1); + + auto mul_sin = make_shared(concat, sin); + auto mul_cos = make_shared(data->output(1), cos); + auto add = make_shared(mul_cos, mul_sin); + + model = std::make_shared(NodeVector{add}, ParameterVector{data_, sin_, cos_}); + + manager.register_pass(); + } +} \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index e979270fee3318..3cc88d43125260 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -30,6 +30,7 @@ #include "transformations/common_optimizations/common_optimizations.hpp" #include "transformations/common_optimizations/wrap_interpolate_into_transposes.hpp" #include "transformations/common_optimizations/matmul_const_transposes_extraction.hpp" +#include "transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp" #include "transformations/control_flow/unroll_tensor_iterator.hpp" #include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/op_conversions/convert_batch_to_space.hpp" @@ -293,6 +294,7 @@ void Transformations::PreLpt(const std::vector& defaultPrecis CPU_REGISTER_PASS_COMMON(manager, ov::pass::AUGRUCellFusion); CPU_REGISTER_PASS_COMMON(manager, ov::pass::CommonOptimizations); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::RPE_Fusion); CPU_REGISTER_PASS_COMMON(manager, ov::pass::WrapInterpolateIntoTransposes); CPU_REGISTER_PASS_COMMON(manager, ov::pass::TransposeSinking); CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertSequenceToTensorIterator); @@ -433,6 +435,7 @@ void Transformations::PreLpt(const std::vector& defaultPrecis CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertTopK11ToTopK3); CPU_DISABLE_PASS_COMMON(manager, ov::pass::HSwishDecomposition); CPU_DISABLE_PASS_COMMON(manager, ov::pass::MatMulConstTransposesExtraction); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::RPE_Fusion); // should be disabled until CPU supports this internal op CPU_DISABLE_PASS_X64(manager, ov::pass::HSigmoidDecomposition); CPU_DISABLE_PASS_X64(manager, ov::pass::ReduceL1Decomposition); From ec25f90ba01272168c2577b33115ec07cfa855c7 Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Fri, 27 Oct 2023 14:33:03 +0400 Subject: [PATCH 2/9] Correct includes --- .../ov_ops/rotary_positional_embeddings.hpp | 4 ---- .../fuse_rotary_positional_embeddings.hpp | 5 ++--- .../fuse_rotary_positional_embeddings.cpp | 18 ++++++++---------- .../fuse_rotary_positional_embeddings.cpp | 12 ++++++------ 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp index 2a03637f6f7383..2fdb6389612e54 100644 --- a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp +++ b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp @@ -4,10 +4,6 @@ #pragma once -#include -#include -#include - #include "openvino/op/op.hpp" #include "transformations_visibility.hpp" diff --git a/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp b/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp index 746b0e41ec58ee..9c756b48896f62 100644 --- a/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp +++ b/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp @@ -4,9 +4,8 @@ #pragma once -#include -#include -#include +#include "openvino/pass/graph_rewrite.hpp" +#include "transformations_visibility.hpp" namespace ov { namespace pass { diff --git a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp index e3d6b3f7a9d5e6..57c6f260f06bf1 100644 --- a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -4,17 +4,15 @@ #include "transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "itt.hpp" +#include "openvino/core/validation_util.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/op.hpp" +#include "openvino/op/variadic_split.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "ov_ops/rotary_positional_embeddings.hpp" #include "transformations/utils/utils.hpp" ov::pass::RPE_Fusion::RPE_Fusion() { diff --git a/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp index 584b46c7fd222c..94e5375295f087 100644 --- a/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -2,15 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 // -#include +#include "transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp" -#include -#include -#include -#include -#include +#include #include "common_test_utils/ov_test_utils.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/variadic_split.hpp" +#include "ov_ops/rotary_positional_embeddings.hpp" +#include "transformations/utils/utils.hpp" using namespace std; using namespace ov; From beee26ac4fdaf710b67d994471aeaa8072696f0a Mon Sep 17 00:00:00 2001 From: Evgenya Nugmanova Date: Mon, 13 Nov 2023 14:40:37 +0400 Subject: [PATCH 3/9] Apply suggestions from code review Co-authored-by: Pawel Raasz --- .../include/ov_ops/rotary_positional_embeddings.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp index 2fdb6389612e54..6757c18bcfd7b8 100644 --- a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp +++ b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp @@ -20,7 +20,7 @@ class TRANSFORMATIONS_API RPE : public ov::op::Op { OPENVINO_OP("RPE", "ie_internal_opset", op::Op); RPE() = default; - RPE(const Output& data, const Output& sin, const Output& cos, const int64_t& axis); + RPE(const Output& data, const Output& sin, const Output& cos, const int64_t axis); bool visit_attributes(AttributeVisitor& visitor) override; void validate_and_infer_types() override; @@ -29,7 +29,7 @@ class TRANSFORMATIONS_API RPE : public ov::op::Op { int64_t get_axis() const { return m_axis; }; - void set_axis(const int64_t& axis) { + void set_axis(const int64_t axis) { m_axis = axis; }; From 48b2f389cabf2cb4d6a00881cb149daee8d8980f Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Wed, 15 Nov 2023 12:32:06 +0400 Subject: [PATCH 4/9] Comments adressed --- .../ov_ops/rotary_positional_embeddings.hpp | 10 +++------ .../ov_ops/rotary_positional_embeddings.cpp | 21 +++++++++++++++---- .../fuse_rotary_positional_embeddings.cpp | 19 ++++++++++------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp index 6757c18bcfd7b8..31efe5d6abbaec 100644 --- a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp +++ b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp @@ -22,17 +22,13 @@ class TRANSFORMATIONS_API RPE : public ov::op::Op { RPE() = default; RPE(const Output& data, const Output& sin, const Output& cos, const int64_t axis); + void set_axis(const int64_t axis); + int64_t get_axis() const; + bool visit_attributes(AttributeVisitor& visitor) override; void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - int64_t get_axis() const { - return m_axis; - }; - void set_axis(const int64_t axis) { - m_axis = axis; - }; - private: int64_t m_axis{}; }; diff --git a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp index accd8a038e8baa..44e7462a8d0bd8 100644 --- a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp @@ -6,15 +6,24 @@ #include "itt.hpp" -using namespace std; -using namespace ov::op::internal; +namespace ov { +namespace op { +namespace internal { -RPE::RPE(const Output& data, const Output& sin, const Output& cos, const int64_t& axis) +RPE::RPE(const Output& data, const Output& sin, const Output& cos, const int64_t axis) : Op({data, sin, cos}), m_axis{axis} { constructor_validate_and_infer_types(); } +void RPE::set_axis(const int64_t axis) { + m_axis = axis; +} + +int64_t RPE::get_axis() const { + return m_axis; +} + void RPE::validate_and_infer_types() { INTERNAL_OP_SCOPE(internal_RoPE_validate_and_infer_types); set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); @@ -26,7 +35,11 @@ bool RPE::visit_attributes(ov::AttributeVisitor& visitor) { return true; } -shared_ptr RPE::clone_with_new_inputs(const ov::OutputVector& new_args) const { +std::shared_ptr RPE::clone_with_new_inputs(const ov::OutputVector& new_args) const { INTERNAL_OP_SCOPE(internal_RoPE_clone_with_new_inputs); return make_shared(new_args.at(0), new_args.at(1), new_args.at(2), m_axis); } + +} // namespace internal +} // namespace op +} // namespace ovqcleaq \ No newline at end of file diff --git a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp index 57c6f260f06bf1..91704105358918 100644 --- a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -15,6 +15,11 @@ #include "ov_ops/rotary_positional_embeddings.hpp" #include "transformations/utils/utils.hpp" +using ov::op::v0::Concat; +using ov::op::v1::Add; +using ov::op::v1::Multiply; +using ov::op::v1::VariadicSplit; + ov::pass::RPE_Fusion::RPE_Fusion() { MATCHER_SCOPE(RPE_Fusion); @@ -29,21 +34,21 @@ ov::pass::RPE_Fusion::RPE_Fusion() { // Variadic Split into two equal parts auto axis = pattern::any_input(); auto split_length = INT_CONSTANT_WITH_PREDICATE(value.size() == 2 && value[0] == value[1]); - auto vsplit = pattern::wrap_type({source, axis, split_length}); + auto vsplit = pattern::wrap_type({source, axis, split_length}); vsplit->set_output_size(2); // Negate auto minus_1 = FLOAT_CONSTANT_WITH_PREDICATE(value.size() == 1 && value[0] == -1); - auto neg = pattern::wrap_type({vsplit->output(1), minus_1}); + auto neg = pattern::wrap_type({vsplit->output(1), minus_1}); // Concat two splitted parts in the opposite order, first of them is negated - auto concat = pattern::wrap_type({neg, vsplit->output(0)}); // make sure axis eq to vsplit eq -1 + auto concat = pattern::wrap_type({neg, vsplit->output(0)}); // make sure axis eq to vsplit eq -1 // END: rotate half - auto mul_sin = pattern::wrap_type({concat, sin}); - auto mul_cos = pattern::wrap_type({source_1, cos}); - auto add = pattern::wrap_type({mul_cos, mul_sin}); + auto mul_sin = pattern::wrap_type({concat, sin}); + auto mul_cos = pattern::wrap_type({source_1, cos}); + auto add = pattern::wrap_type({mul_cos, mul_sin}); ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { auto value_map = m.get_pattern_value_map(); @@ -60,7 +65,7 @@ ov::pass::RPE_Fusion::RPE_Fusion() { cos_output = potential_source; auto input = value_map.at(source); - auto concat_node = ov::as_type_ptr(value_map.at(concat).get_node_shared_ptr()); + auto concat_node = ov::as_type_ptr(value_map.at(concat).get_node_shared_ptr()); if (!concat_node) return false; OPENVINO_SUPPRESS_DEPRECATED_START From 4ec92bff04730f3fa57b7411165328ab93aace45 Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Wed, 15 Nov 2023 12:37:18 +0400 Subject: [PATCH 5/9] Misprints --- .../include/ov_ops/rotary_positional_embeddings.hpp | 4 ++-- .../src/ov_ops/rotary_positional_embeddings.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp index 31efe5d6abbaec..0ae6db9ddcc3ad 100644 --- a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp +++ b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp @@ -20,9 +20,9 @@ class TRANSFORMATIONS_API RPE : public ov::op::Op { OPENVINO_OP("RPE", "ie_internal_opset", op::Op); RPE() = default; - RPE(const Output& data, const Output& sin, const Output& cos, const int64_t axis); + RPE(const Output& data, const Output& sin, const Output& cos, int64_t axis); - void set_axis(const int64_t axis); + void set_axis(int64_t axis); int64_t get_axis() const; bool visit_attributes(AttributeVisitor& visitor) override; diff --git a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp index 44e7462a8d0bd8..ed4ee7850ef050 100644 --- a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp @@ -10,13 +10,13 @@ namespace ov { namespace op { namespace internal { -RPE::RPE(const Output& data, const Output& sin, const Output& cos, const int64_t axis) +RPE::RPE(const Output& data, const Output& sin, const Output& cos, int64_t axis) : Op({data, sin, cos}), m_axis{axis} { constructor_validate_and_infer_types(); } -void RPE::set_axis(const int64_t axis) { +void RPE::set_axis(int64_t axis) { m_axis = axis; } @@ -37,9 +37,9 @@ bool RPE::visit_attributes(ov::AttributeVisitor& visitor) { std::shared_ptr RPE::clone_with_new_inputs(const ov::OutputVector& new_args) const { INTERNAL_OP_SCOPE(internal_RoPE_clone_with_new_inputs); - return make_shared(new_args.at(0), new_args.at(1), new_args.at(2), m_axis); + return std::make_shared(new_args.at(0), new_args.at(1), new_args.at(2), m_axis); } } // namespace internal } // namespace op -} // namespace ovqcleaq \ No newline at end of file +} // namespace ov \ No newline at end of file From 6d516b5012b6d04f8f52feadf72bf435da764419 Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Wed, 6 Dec 2023 15:46:27 +0400 Subject: [PATCH 6/9] Ivan comments adressed --- .../src/ov_ops/rotary_positional_embeddings.cpp | 1 + .../intel_cpu/src/transformations/transformation_pipeline.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp index ed4ee7850ef050..02f16f01ce5298 100644 --- a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp @@ -37,6 +37,7 @@ bool RPE::visit_attributes(ov::AttributeVisitor& visitor) { std::shared_ptr RPE::clone_with_new_inputs(const ov::OutputVector& new_args) const { INTERNAL_OP_SCOPE(internal_RoPE_clone_with_new_inputs); + check_new_args_count(this, new_args); return std::make_shared(new_args.at(0), new_args.at(1), new_args.at(2), m_axis); } diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index c9a50b1cca943c..c8e0974ca49487 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -475,7 +475,8 @@ void Transformations::PreLpt(const std::vector& defaultPrecis CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertTopK11ToTopK3); CPU_DISABLE_PASS_COMMON(manager, ov::pass::HSwishDecomposition); CPU_DISABLE_PASS_COMMON(manager, ov::pass::MatMulConstTransposesExtraction); - CPU_DISABLE_PASS_COMMON(manager, ov::pass::RPE_Fusion); // should be disabled until CPU supports this internal op + // CVS-126827: should be disabled until CPU supports this internal op + CPU_DISABLE_PASS_COMMON(manager, ov::pass::RPE_Fusion); CPU_DISABLE_PASS_X64(manager, ov::pass::HSigmoidDecomposition); CPU_DISABLE_PASS_X64(manager, ov::pass::ReduceL1Decomposition); From 86537124c3b2f1047944181a6e0f910cf11aa29a Mon Sep 17 00:00:00 2001 From: Evgenya Nugmanova Date: Wed, 6 Dec 2023 15:43:35 +0400 Subject: [PATCH 7/9] Update src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp Co-authored-by: Pawel Raasz --- .../common_optimizations/fuse_rotary_positional_embeddings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp index 91704105358918..b452c636395304 100644 --- a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -69,7 +69,7 @@ ov::pass::RPE_Fusion::RPE_Fusion() { if (!concat_node) return false; OPENVINO_SUPPRESS_DEPRECATED_START - auto split_axis_node = ov::get_constant_from_source(value_map.at(axis)); + auto split_axis_node = ov::util::get_constant_from_source(value_map.at(axis)); OPENVINO_SUPPRESS_DEPRECATED_END if (!split_axis_node) return false; From b684d5c186fe2a333354c5169e8e4415d645602a Mon Sep 17 00:00:00 2001 From: Evgenya Nugmanova Date: Wed, 6 Dec 2023 15:43:40 +0400 Subject: [PATCH 8/9] Update src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp Co-authored-by: Pawel Raasz --- .../fuse_rotary_positional_embeddings.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp index b452c636395304..b5fe9a62d65f39 100644 --- a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -81,8 +81,8 @@ ov::pass::RPE_Fusion::RPE_Fusion() { if (concat_axis != split_axis) { if (input.get_partial_shape().is_static()) { auto rank = input.get_partial_shape().rank().get_length(); - concat_axis = concat_axis < 0 ? concat_axis + rank : concat_axis; - split_axis = split_axis < 0 ? split_axis + rank : split_axis; + concat_axis = ov::util:::normalize(concat_axis, rank); + split_axis = ov::util:::normalize(split_axis, rank); } if (concat_axis != split_axis) return false; From 60459380ee3083ebf7e95e0d39413852fcd8aff4 Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Wed, 6 Dec 2023 16:36:35 +0400 Subject: [PATCH 9/9] Fix includes and adds comments --- .../fuse_rotary_positional_embeddings.cpp | 15 ++++++++------- .../fuse_rotary_positional_embeddings.cpp | 16 +++++++++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp index b5fe9a62d65f39..689664922486b7 100644 --- a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -14,6 +14,7 @@ #include "openvino/pass/pattern/op/wrap_type.hpp" #include "ov_ops/rotary_positional_embeddings.hpp" #include "transformations/utils/utils.hpp" +#include "validation_util.hpp" using ov::op::v0::Concat; using ov::op::v1::Add; @@ -68,9 +69,7 @@ ov::pass::RPE_Fusion::RPE_Fusion() { auto concat_node = ov::as_type_ptr(value_map.at(concat).get_node_shared_ptr()); if (!concat_node) return false; - OPENVINO_SUPPRESS_DEPRECATED_START auto split_axis_node = ov::util::get_constant_from_source(value_map.at(axis)); - OPENVINO_SUPPRESS_DEPRECATED_END if (!split_axis_node) return false; auto value = split_axis_node->cast_vector(); @@ -79,18 +78,20 @@ ov::pass::RPE_Fusion::RPE_Fusion() { auto concat_axis = concat_node->get_concatenation_axis(); auto split_axis = value[0]; if (concat_axis != split_axis) { - if (input.get_partial_shape().is_static()) { + if (input.get_partial_shape().rank().is_static()) { auto rank = input.get_partial_shape().rank().get_length(); - concat_axis = ov::util:::normalize(concat_axis, rank); - split_axis = ov::util:::normalize(split_axis, rank); + concat_axis = ov::util::normalize(concat_axis, rank); + split_axis = ov::util::normalize(split_axis, rank); } if (concat_axis != split_axis) return false; } auto rpe = std::make_shared(input, value_map.at(sin), cos_output, concat_node->get_axis()); - ov::replace_output_update_name(value_map.at(add), rpe->output(0)); - return true; + + for (const auto& label : {vsplit, neg, concat, mul_sin, mul_cos, add}) + ov::copy_runtime_info(value_map.at(label).get_node_shared_ptr(), rpe); + return ov::replace_output_update_name(value_map.at(add), rpe->output(0)); }; auto m = std::make_shared(add, matcher_name); register_matcher(m, matcher_pass_callback); diff --git a/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp index 94e5375295f087..48ee524c1fb182 100644 --- a/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -62,14 +62,20 @@ TEST_F(TransformationTestsF, FuseRPE) { } TEST_F(TransformationTestsF, FuseRPESorcesAreMultiOutputed) { + /* Transformation matcher searches for a single source as a beginning of the pattern: + VariadicSplit ... + source ____/ + \ + Multiply ... + This test is designed to check that in case we feed VariadicSplit and Multiply from different outputs of the same + node, the transformation won't happen since the source isn't the same + */ { auto data_ = make_shared(element::f32, PartialShape::dynamic()); - auto sin_ = make_shared(element::f32, PartialShape::dynamic()); - auto cos_ = make_shared(element::f32, PartialShape::dynamic()); + auto sin = make_shared(element::f32, PartialShape::dynamic()); + auto cos = make_shared(element::f32, PartialShape::dynamic()); auto data = make_shared(data_, v0::Constant::create(element::i64, {}, {-1}), 2); - auto sin = make_shared(sin_, v0::Constant::create(element::i64, {}, {-1}), 2)->output(1); - auto cos = make_shared(cos_, v0::Constant::create(element::i64, {}, {-1}), 2)->output(1); auto axis = v0::Constant::create(element::i64, {}, {-1}); auto split_lengths = v0::Constant::create(element::i64, {2}, {10, 10}); @@ -84,7 +90,7 @@ TEST_F(TransformationTestsF, FuseRPESorcesAreMultiOutputed) { auto mul_cos = make_shared(data->output(1), cos); auto add = make_shared(mul_cos, mul_sin); - model = std::make_shared(NodeVector{add}, ParameterVector{data_, sin_, cos_}); + model = std::make_shared(NodeVector{add}, ParameterVector{data_, sin, cos}); manager.register_pass(); }