Merge branch 'master' into pfn_build_log

.github/workflows/build_doc.yml

@@ -63,7 +63,7 @@ jobs:
 
       - name: Cache documentation
         id: cache_sphinx_docs
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         with:
           path: build/docs/_build/.doctrees
           key: sphinx-docs-cache

.github/workflows/job_cpu_functional_tests.yml

@@ -89,7 +89,7 @@ jobs:
         run: python3 -m pip install -r ${INSTALL_TEST_DIR}/functional_test_utils/layer_tests_summary/requirements.txt
 
       - name: Restore tests execution time
-        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache/restore@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}
           key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }}
@@ -109,7 +109,7 @@ jobs:
         timeout-minutes: 25
 
       - name: Save tests execution time
-        uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache/save@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         if: github.ref_name == 'master'
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}

.github/workflows/mo.yml

@@ -32,7 +32,7 @@ jobs:
           python-version: '3.10'
 
       - name: Cache pip
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('tools/mo/requirements*.txt') }}

.github/workflows/ovc.yml

@@ -27,7 +27,7 @@ jobs:
           python-version: '3.10'
 
       - name: Cache pip
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('src/bindings/python/requirements*.txt') }}

.github/workflows/windows_conditional_compilation.yml

@@ -387,7 +387,7 @@ jobs:
         run: python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/layer_tests_summary/requirements.txt
 
       - name: Restore tests execution time
-        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache/restore@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}
           key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }}

.github/workflows/windows_vs2019_release.yml

@@ -481,7 +481,7 @@ jobs:
         run: python3 -m pip install -r ${{ github.workspace }}\install\tests\functional_test_utils\layer_tests_summary\requirements.txt
 
       - name: Restore tests execution time
-        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache/restore@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}
           key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }}
@@ -495,7 +495,7 @@ jobs:
         timeout-minutes: 60
 
       - name: Save tests execution time
-        uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache/save@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
         if: github.ref_name == 'master'
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}

docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst

@@ -197,6 +197,7 @@ Operation Specifications
    ScatterElementsUpdate-12 <operation-specs/movement/scatter-elements-update-12>
    ScatterNDUpdate-3 <operation-specs/movement/scatter-nd-update-3>
    ScatterUpdate-3 <operation-specs/movement/scatter-update-3>
+   SearchSorted-15 <operation-specs/sort/search-sorted-15>
    Select-1 <operation-specs/condition/select-1>
    Selu-1 <operation-specs/activation/selu-1>
    ShapeOf-1 <operation-specs/shape/shape-of-1>

docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/search-sorted-15.rst

@@ -0,0 +1,73 @@
+SearchSorted
+===============
+
+
+.. meta::
+  :description: Learn about SearchSorted - a sorting and maximization
+                operation, which requires two input tensors.
+
+
+**Versioned name**: *SearchSorted-15*
+
+**Category**: *Sorting and maximization*
+
+**Short description**: Determines the indices in the innermost dimension of a sorted sequence where elements should be inserted to maintain order.
+
+**Detailed description**: *SearchSorted* operation determines the indices in the innermost dimension of a sorted sequence where elements should be inserted to maintain order. The operation is based on the binary search algorithm. The operation is performed on two input tensors: the first tensor contains a monotonically increasing sequence on the innermost dimension, and the second tensor contains the search values. The operation returns a tensor with the same shape as the second input tensor, containing the indices.
+
+**Attributes**
+
+* *right*
+
+  * **Description**: If False, set the first suitable index. If True, return the last suitable index for given value. Default is False.
+  * **Range of values**: true or false
+  * **Type**: boolean
+  * **Default value**: false
+  * **Required**: *no*
+
+**Inputs**:
+
+* **1**: ``sorted`` - ND input tensor of type *T* - cannot be a scalar, containing monotonically increasing sequence on the innermost dimension. **Required.**
+
+* **2**: ``values`` - ND input tensor of type *T*, containing the search values. If sorted sequence is 1D, then the values can have any shape, otherwise the rank should be equal to the rank of sorted input. **Required.**
+
+**Outputs**:
+
+* **1**: Tensor of type *TOut*, with the same shape as second input tensor, containing the indices.
+
+**Types**
+
+* *T*: any supported floating-point and integer type.
+
+* *TOut*: int64.
+
+**Example**
+
+.. code-block:: xml
+   :force:
+
+   <layer ... type="SearchSorted" ... >
+       <data right="True"/>
+       <input>
+           <port id="0">
+               <dim>7</dim>
+               <dim>256</dim>
+               <dim>200</dim>
+               <dim>200</dim>
+           </port>
+           <port id="1">
+               <dim>7</dim>
+               <dim>256</dim>
+               <dim>200</dim>
+               <dim>10</dim>
+           </port>
+       </input>
+       <output>
+           <port id="0" precision="I64">
+               <dim>7</dim>
+               <dim>256</dim>
+               <dim>200</dim>
+               <dim>10</dim>
+           </port>
+       </output>
+   </layer>

src/bindings/python/constraints.txt

@@ -10,7 +10,7 @@ pytest-timeout==2.3.1
 # Python bindings
 py>=1.9.0
 pygments>=2.8.1
-setuptools>=65.6.1,<74.1.0
+setuptools>=65.6.1,<75.3.0
 sympy>=1.10
 wheel>=0.38.1
 patchelf<=0.17.2.1

src/bindings/python/src/openvino/runtime/opset15/__init__.py

@@ -15,3 +15,4 @@
 from openvino.runtime.opset15.ops import string_tensor_unpack
 from openvino.runtime.opset15.ops import bitwise_left_shift
 from openvino.runtime.opset15.ops import bitwise_right_shift
+from openvino.runtime.opset15.ops import slice_scatter

src/bindings/python/src/openvino/runtime/opset15/ops.py

@@ -274,3 +274,32 @@ def bitwise_right_shift(
             "auto_broadcast": auto_broadcast.upper(),
         },
     )
+
+
+@nameable_op
+def slice_scatter(
+    data: NodeInput,
+    updates: NodeInput,
+    start: NodeInput,
+    stop: NodeInput,
+    step: NodeInput,
+    axes: Optional[NodeInput] = None,
+    name: Optional[str] = None,
+) -> Node:
+    """Return a node which generates SliceScatter operation.
+
+    :param  data: The node providing input data.
+    :param  updates: The node providing updates data.
+    :param  start: The node providing start indices (inclusively).
+    :param  stop: The node providing stop indices (exclusively).
+    :param  step: The node providing step values.
+    :param  axes: The optional node providing axes to slice, default [0, 1, ..., len(start)-1].
+    :param  name: The optional name for the created output node.
+    :return: The new node performing SliceScatter operation.
+    """
+    if axes is None:
+        inputs = as_nodes(data, updates, start, stop, step, name=name)
+    else:
+        inputs = as_nodes(data, updates, start, stop, step, axes, name=name)
+
+    return _get_node_factory_opset15().create("SliceScatter", inputs)

src/bindings/python/tests/test_graph/test_create_op.py

@@ -2458,6 +2458,34 @@ def test_topk_opset11(op_name):
     assert list(node.get_output_shape(1)) == [1, 3, 3]
 
 
+def test_slice_scatter():
+    data_shape = [10, 7, 2, 13]
+    data = ov.parameter(data_shape, name="input", dtype=np.float32)
+    updates = ov.parameter([4, 7, 2, 13], name="updates", dtype=np.float32)
+    start = ov.constant(np.array([2, 0, 0], dtype=np.int32))
+    stop = ov.constant(np.array([9, 7, 2], dtype=np.int32))
+    step = ov.constant(np.array([2, 1, 1], dtype=np.int32))
+
+    node_default_axes = ov_opset15.slice_scatter(data, updates, start, stop, step)
+
+    assert node_default_axes.get_type_name() == "SliceScatter"
+    assert node_default_axes.get_output_size() == 1
+    assert node_default_axes.get_output_element_type(0) == Type.f32
+    assert node_default_axes.get_output_shape(0) == data_shape
+
+    start = ov.constant(np.array([0, 2], dtype=np.int32))
+    stop = ov.constant(np.array([2, 9], dtype=np.int32))
+    step = ov.constant(np.array([1, 2], dtype=np.int32))
+    axes = ov.constant(np.array([-2, 0], dtype=np.int32))
+
+    node = ov_opset15.slice_scatter(data, updates, start, stop, step, axes)
+
+    assert node.get_type_name() == "SliceScatter"
+    assert node.get_output_size() == 1
+    assert node.get_output_element_type(0) == Type.f32
+    assert node_default_axes.get_output_shape(0) == data_shape
+
+
 def test_parameter_get_attributes():
     parameter = ov.parameter([2, 2], dtype=np.float32, name="InputData")
     parameter_attributes = parameter.get_attributes()

src/common/transformations/src/transformations/sdpa_to_paged_attention/state_management_pattern.cpp

@@ -383,12 +383,18 @@ ov::pass::StateManagementPattern::StateManagementPattern(ParameterVector& kv_par
 
         auto paged_attention = std::make_shared<ov::op::PagedAttentionExtension>(pa_arguments);
 
+        // The output shape of PagedAttention will be converted to [batch, 1, head_num, head_size_v], the head_size_v
+        // may be different from head_size_q/head_size_k. The head_size_v could be got from the shape of value input
+        auto hidden_dim_v = std::make_shared<v8::Gather>(std::make_shared<v3::ShapeOf>(v_target_layout),
+                                                         v0::Constant::create(element::i64, Shape{}, {-1}),
+                                                         v0::Constant::create(element::i64, Shape{}, {0}));
+
         auto pa_shape = std::make_shared<v0::Concat>(
             OutputVector{
                 v0::Constant::create(element::i64, Shape{1}, {0}),
                 v0::Constant::create(element::i64, Shape{1}, {1}),
                 v0::Constant::create(element::i64, Shape{1}, {-1}),
-                std::make_shared<v0::Unsqueeze>(hidden_dim, v0::Constant::create(element::i64, Shape{}, {0})),
+                std::make_shared<v0::Unsqueeze>(hidden_dim_v, v0::Constant::create(element::i64, Shape{}, {0})),
             },
             0);
         auto pa_reshape = std::make_shared<v1::Reshape>(paged_attention->output(0), pa_shape, true);

src/core/src/op/paged_attention.cpp

@@ -4,6 +4,7 @@
 
 #include "openvino/op/paged_attention.hpp"
 
+#include "dimension_util.hpp"
 #include "itt.hpp"
 #include "openvino/op/op.hpp"
 
@@ -146,10 +147,33 @@ void PagedAttentionExtension::validate_and_infer_types() {
                           get_input_element_type(12),
                           ".");
 
+    // value head_size may be not same with key
+    auto out_ps = get_input_partial_shape(0);
+    const auto& key_ps = get_input_partial_shape(1);
+    const auto& value_ps = get_input_partial_shape(2);
+    if (out_ps.rank().is_static()) {
+        if (key_ps.rank().is_static() && value_ps.rank().is_static() && key_ps[1].is_static()) {
+            // The dim of out_ps[1] should be `num_heads * v_head_size`, it can be got from:
+            // because:
+            //   q: query_ps[1] = num_heads * head_size
+            //   k: key_ps[1] = num_kv_heads * head_size
+            //   v: value_ps[1] = num_kv_heads * v_head_size
+            // therefore:
+            //   q * v / k = (num_heads * head_size) * (num_kv_heads * v_head_size) /
+            //               (num_kv_heads * head_size) = num_heads * v_head_size
+            out_ps[1] = out_ps[1] * value_ps[1] / key_ps[1].get_length();
+            NODE_VALIDATION_CHECK(this,
+                                  !ov::util::dim::is_empty(out_ps[1]),
+                                  "The last dimension of output should not be empty.");
+        } else {
+            out_ps[1] = Dimension::dynamic();
+        }
+    }
+
     if (m_output_type[0] == ov::element::undefined) {
-        set_output_type(0, get_input_element_type(0), get_input_partial_shape(0));
+        set_output_type(0, get_input_element_type(0), out_ps);
     } else {
-        set_output_type(0, m_output_type[0], get_input_partial_shape(0));
+        set_output_type(0, m_output_type[0], out_ps);
     }
 
     if (m_output_type[1] == ov::element::undefined) {

src/frontends/pytorch/src/node_context.cpp

@@ -4,6 +4,7 @@
 
 #include "openvino/frontend/pytorch/node_context.hpp"
 
+#include "helper_ops/internal_op.hpp"
 #include "openvino/core/validation_util.hpp"
 #include "openvino/frontend/exception.hpp"
 #include "openvino/frontend/pytorch/decoder.hpp"
@@ -151,13 +152,26 @@ OutputVector NodeContext::inputs() const {
         if (input == 0) {
             // Case when input can be inlined (possible only for fx decoder)
             if (m_decoder->is_input_inlined(i)) {
-                auto inlined_input = m_decoder->inlined_input(i);
-                FRONT_END_GENERAL_CHECK(inlined_input.size() == 1,
-                                        "Incorrect inlined input with index: ",
-                                        i,
-                                        " for operation ",
-                                        get_op_type());
-                res.push_back(inlined_input[0]);
+                if (input_is_none(i)) {
+                    // some operations like aten.index.Tensor can have None inputs
+                    auto dummy_decoder = std::make_shared<InternalOpDecoder>("torch::None", 1);
+                    auto fw_node = std::make_shared<PtFrameworkNode>(dummy_decoder, OutputVector{});
+                    auto attrs = fw_node->get_attrs();
+                    attrs["none_value"] = "";
+                    attrs[PtFrameworkNode::failed_conversion_key] =
+                        "None constant cannot be converted to OpenVINO opset and should be removed by consuming "
+                        "operation.";
+                    fw_node->set_attrs(attrs);
+                    res.push_back(fw_node->output(0));
+                } else {
+                    auto inlined_input = m_decoder->inlined_input(i);
+                    FRONT_END_GENERAL_CHECK(inlined_input.size() == 1,
+                                            "Incorrect inlined input with index: ",
+                                            i,
+                                            " for operation ",
+                                            get_op_type());
+                    res.push_back(inlined_input[0]);
+                }
                 continue;
             }
         }