[GPU] Dynamic quantization for OneDNN FC

src/inference/include/openvino/runtime/properties.hpp

@@ -571,7 +571,7 @@ static constexpr Property<ExecutionMode> execution_mode{"EXECUTION_MODE_HINT"};
  * might result in better accuracy, but the drawback is worse performance. Group size equal 0 means dynamic
  * quantization optimization is disabled.
  */
-static constexpr Property<uint64_t, PropertyMutability::RW> dynamic_quantization_group_size{
+static constexpr Property<int64_t, PropertyMutability::RW> dynamic_quantization_group_size{
     "DYNAMIC_QUANTIZATION_GROUP_SIZE"};
 
 /**

src/plugins/intel_gpu/include/intel_gpu/op/dynamic_quantize.hpp

@@ -0,0 +1,37 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/op/op.hpp"
+
+namespace ov {
+namespace intel_gpu {
+namespace op {
+
+/// \brief Operator performing Dynamic Quantize
+class DynamicQuantize : public ov::op::Op {
+public:
+    OPENVINO_OP("DynamicQuantize", "gpu_opset");
+
+    DynamicQuantize() = default;
+    /// \brief Constructs an DynamicQuantize operation.
+    ///
+    /// \param data Input tensor with data
+    DynamicQuantize(const Output<Node>& data, int64_t group_size);
+
+    void validate_and_infer_types() override;
+
+    std::shared_ptr<Node> clone_with_new_inputs(const ov::OutputVector& new_args) const override;
+    int64_t get_group_size() { return m_group_size; };
+
+private:
+    int64_t m_group_size;
+};
+
+std::vector<ov::PartialShape> shape_infer(const DynamicQuantize* op, std::vector<ov::PartialShape> input_shapes);
+
+}   // namespace op
+}   // namespace intel_gpu
+}   // namespace ov

src/plugins/intel_gpu/include/intel_gpu/op/fully_connected_compressed.hpp

@@ -29,7 +29,20 @@ class FullyConnectedCompressed : public FullyConnected {
                              const ov::Output<Node> &decompression_scale,
                              const ov::element::Type output_type = ov::element::undefined);
 
+    FullyConnectedCompressed(const OutputVector& inputs,
+                             bool has_zp = true,
+                             bool has_activation_scale = false,
+                             const ov::element::Type output_type = ov::element::undefined);
+
     std::shared_ptr<Node> clone_with_new_inputs(const ov::OutputVector& new_args) const override;
+
+    bool get_has_zp() const { return m_has_zp; }
+    bool get_has_activation_scale() const { return m_has_activation_scale; }
+
+
+protected:
+    bool m_has_zp;
+    bool m_has_activation_scale;
 };
 
 }   // namespace op

src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp

@@ -287,3 +287,4 @@ REGISTER_FACTORY(internal, Placeholder);
 REGISTER_FACTORY(internal, SDPA);
 REGISTER_FACTORY(internal, IndirectSDPA);
 REGISTER_FACTORY(internal, RoPE);
+REGISTER_FACTORY(internal, DynamicQuantize);

src/plugins/intel_gpu/include/intel_gpu/primitives/dynamic_quantize.hpp

@@ -0,0 +1,57 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+#include "primitive.hpp"
+
+namespace cldnn {
+
+/// @brief Dynamic Quantize primitive
+/// @details Performs dynamic quantization
+struct dynamic_quantize : public primitive_base<dynamic_quantize> {
+    CLDNN_DECLARE_PRIMITIVE(dynamic_quantize);
+
+    dynamic_quantize() : primitive_base("", {}), group_size(0) {}
+
+    /// @brief Constructs dynamic_quantize primitive
+    /// @param id This primitive id
+    /// @param input Input primitive id
+    /// @param group_size Quantization group size
+    /// @param data_type Output data type of quantized
+    /// @param output_size Output data size of the primitive
+    dynamic_quantize(const primitive_id& id,
+           const input_info& input,
+           const int64_t group_size,
+           const std::vector<optional_data_type> data_types = {optional_data_type(data_types::f16), optional_data_type(data_types::i8)})
+           : primitive_base(id, {input}, 2, data_types)
+	   , group_size(group_size) {}
+
+    int64_t group_size = 0;
+
+    size_t hash() const override {
+        size_t seed = primitive::hash();
+        seed = hash_combine(seed, group_size);
+        return seed;
+    }
+
+    bool operator==(const primitive& rhs) const override {
+        if (!compare_common_params(rhs))
+            return false;
+
+        auto rhs_casted = downcast<const dynamic_quantize>(rhs);
+
+        return group_size == rhs_casted.group_size;
+    }
+
+    void save(BinaryOutputBuffer& ob) const override {
+        primitive_base<dynamic_quantize>::save(ob);
+        ob << group_size;
+    }
+
+    void load(BinaryInputBuffer& ib) override {
+        primitive_base<dynamic_quantize>::load(ib);
+        ib >> group_size;
+    }
+};
+}  // namespace cldnn

src/plugins/intel_gpu/include/intel_gpu/primitives/fully_connected.hpp

@@ -95,11 +95,46 @@ struct fully_connected : public primitive_base<fully_connected> {
           compressed_weights(true),
           decompression_scale(decompression_scale),
           decompression_zero_point(decompression_zero_point),
+          dynamic_quantized_activation(false),
           input_size(input_size),
           weights_rank(weights_rank) {
         OPENVINO_ASSERT(!decompression_scale.empty(), "[GPU] Compressed fully connected requires at least decompression scale input");
     }
 
+    /// @brief Constructs fully connected compressed layer.
+    /// @param id This primitive id.
+    /// @param input Input primitive id.
+    /// @param weights Primitive id containing weights data.
+    /// @param bias Primitive id containing bias data.
+    /// @param compression_scale Primitive id containing scale factors for weights decompression.
+    /// @param compression_zero_point Primitive id containing zero points for weights decompression.
+    /// @param activation_scale Primitive id containing scale factor for activation.
+    fully_connected(const primitive_id& id,
+                    const input_info& input,
+                    const primitive_id& weights,
+                    const primitive_id& bias,
+                    const primitive_id& decompression_scale,
+                    const primitive_id& decompression_zero_point,
+                    const input_info& activation_scale,
+                    const data_types data_type,
+                    const size_t input_size = 2,
+                    const size_t weights_rank = 2)
+        : primitive_base(id, { input }, 1, {optional_data_type{data_type}}),
+          weights(weights),
+          bias(bias),
+          compressed_weights(true),
+          decompression_scale(decompression_scale),
+          decompression_zero_point(decompression_zero_point),
+          dynamic_quantized_activation(false),
+          activation_scale(activation_scale),
+          input_size(input_size),
+          weights_rank(weights_rank) {
+        if (activation_scale.is_valid())
+            dynamic_quantized_activation = true;
+
+        OPENVINO_ASSERT(!decompression_scale.empty(), "[GPU] Compressed fully connected requires at least decompression scale input");
+    }
+
     /// @brief Primitive id containing weights data.
     primitive_id weights;
     /// @brief Primitive id containing bias data.
@@ -108,6 +143,8 @@ struct fully_connected : public primitive_base<fully_connected> {
     bool compressed_weights = false;
     primitive_id decompression_scale = "";
     primitive_id decompression_zero_point = "";
+    bool dynamic_quantized_activation = false;
+    input_info activation_scale = {"", 0};
     optional_value<float> decompression_zero_point_scalar = optional_value<float>();
 
     /// @brief Primitive dimension size.
@@ -123,6 +160,7 @@ struct fully_connected : public primitive_base<fully_connected> {
         seed = hash_combine(seed, compressed_weights);
         seed = hash_combine(seed, !decompression_scale.empty());
         seed = hash_combine(seed, !decompression_zero_point.empty());
+        seed = hash_combine(seed, activation_scale.is_valid());
         seed = hash_combine(seed, decompression_zero_point_scalar.has_value());
         seed = hash_combine(seed, decompression_zero_point_scalar.value_or(0.0f));
         return seed;
@@ -140,6 +178,7 @@ struct fully_connected : public primitive_base<fully_connected> {
                compressed_weights == rhs_casted.compressed_weights &&
                decompression_scale.empty() == rhs_casted.decompression_scale.empty() &&
                decompression_zero_point.empty() == rhs_casted.decompression_zero_point.empty() &&
+               activation_scale.is_valid() == rhs_casted.activation_scale.is_valid() &&
                decompression_zero_point_scalar.value_or(0.0f) == rhs_casted.decompression_zero_point_scalar.value_or(0.0f);
     }
 
@@ -150,8 +189,10 @@ struct fully_connected : public primitive_base<fully_connected> {
         ob << compressed_weights;
         ob << decompression_scale;
         ob << decompression_zero_point;
+        ob << activation_scale;
         ob << input_size;
         ob << weights_rank;
+        ob << dynamic_quantized_activation;
 
         if (decompression_zero_point_scalar.has_value()) {
             ob << true;
@@ -169,8 +210,10 @@ struct fully_connected : public primitive_base<fully_connected> {
         ib >> compressed_weights;
         ib >> decompression_scale;
         ib >> decompression_zero_point;
+        ib >> activation_scale;
         ib >> input_size;
         ib >> weights_rank;
+        ib >> dynamic_quantized_activation;
 
         bool has_value;
         ib >> has_value;
@@ -197,6 +240,9 @@ struct fully_connected : public primitive_base<fully_connected> {
         if (!decompression_zero_point.empty())
             ret.push_back(decompression_zero_point);
 
+        if (activation_scale.is_valid())
+            ret.push_back(activation_scale);
+
         return ret;
     }
 };

src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp

@@ -0,0 +1,63 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "intel_gpu/op/dynamic_quantize.hpp"
+#include "dynamic_quantize_inst.h"
+
+#include "primitive_type_base.h"
+#include "json_object.h"
+#include <string>
+
+namespace cldnn {
+GPU_DEFINE_PRIMITIVE_TYPE_ID(dynamic_quantize);
+
+layout dynamic_quantize_inst::calc_output_layout(dynamic_quantize_node const& node, kernel_impl_params const& impl_param) {
+    auto desc = impl_param.typed_desc<dynamic_quantize>();
+    auto input_layout = impl_param.get_input_layout();
+    auto output_type = data_types::i8;
+    auto output_format = input_layout.format;
+
+    return layout(output_type, output_format, input_layout.get_tensor());
+}
+
+template<typename ShapeType>
+std::vector<layout> dynamic_quantize_inst::__calc_output_layouts(layout &act_layout, int64_t group_size) {
+    ov::intel_gpu::op::DynamicQuantize op;
+    auto output_format = act_layout.format;
+
+    std::vector<ShapeType> input_shapes = {
+        act_layout.get<ShapeType>(),
+    };
+
+    auto output_shapes = shape_infer(&op, input_shapes);
+
+    return { layout(output_shapes[0], data_types::i8, output_format), layout(output_shapes[1], data_types::f16, output_format) };
+}
+
+template std::vector<layout> dynamic_quantize_inst::__calc_output_layouts<ov::PartialShape>(layout &act_layout, int64_t group_size);
+
+template<typename ShapeType>
+std::vector<layout> dynamic_quantize_inst::calc_output_layouts(dynamic_quantize_node const& /*node*/, const kernel_impl_params& impl_param) {
+    auto desc = impl_param.typed_desc<dynamic_quantize>();
+    auto input_layout = impl_param.get_input_layout();
+    return __calc_output_layouts<ov::PartialShape>(input_layout, 0 /* TODO: handle group_size here */);
+}
+
+template std::vector<layout> dynamic_quantize_inst::calc_output_layouts<ov::PartialShape>(dynamic_quantize_node const& node,
+                                                                                const kernel_impl_params& impl_param);
+
+std::string dynamic_quantize_inst::to_string(dynamic_quantize_node const& node) {
+    auto desc = node.get_primitive();
+    auto node_info = node.desc_to_json();
+
+    std::stringstream primitive_description;
+
+    node_info->dump(primitive_description);
+
+    return primitive_description.str();
+}
+
+dynamic_quantize_inst::typed_primitive_inst(network& network, dynamic_quantize_node const& node) : parent(network, node) {}
+
+}  // namespace cldnn

src/plugins/intel_gpu/src/graph/fully_connected.cpp

@@ -277,6 +277,9 @@ std::string fully_connected_inst::to_string(fully_connected_node const& node) {
             fc_info.add("decompression zp value", desc->decompression_zero_point_scalar.value());
         }
     }
+    if (desc->dynamic_quantized_activation) {
+        fc_info.add("activation scale id", desc->activation_scale.pid);
+    }
 
     node_info->add("fully connected info", fc_info);
     node_info->dump(primitive_description);

src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp

@@ -409,6 +409,8 @@ void prepare_primitive_fusing::fuse_bias(program &p) {
                 fc_with_bias_prim->decompression_zero_point = desc->decompression_zero_point;
                 if (desc->decompression_zero_point_scalar.has_value())
                     fc_with_bias_prim->decompression_zero_point_scalar = desc->decompression_zero_point_scalar.value();
+                fc_with_bias_prim->activation_scale = desc->activation_scale;
+                fc_with_bias_prim->dynamic_quantized_activation = desc->dynamic_quantized_activation;
             }
             auto& new_fc_node = p.get_or_create(fc_with_bias_prim);
             fuse_bias_f(fc, new_fc_node, bias_node, eltw_node);

src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp

@@ -0,0 +1,73 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "openvino/core/validation_util.hpp"
+#include "primitive_base.hpp"
+#include "dynamic_quantize/dynamic_quantize_kernel_ref.h"
+#include "dynamic_quantize/dynamic_quantize_kernel_selector.h"
+#include "dynamic_quantize_inst.h"
+
+namespace cldnn {
+namespace ocl {
+
+struct dynamic_quantize_impl : typed_primitive_impl_ocl<dynamic_quantize> {
+    using parent = typed_primitive_impl_ocl<dynamic_quantize>;
+    using parent::parent;
+    using kernel_selector_t = kernel_selector::dynamic_quantize_kernel_selector;
+    using kernel_params_t = kernel_selector::dynamic_quantize_params;
+
+    DECLARE_OBJECT_TYPE_SERIALIZATION(cldnn::ocl::dynamic_quantize_impl);
+
+    std::unique_ptr<primitive_impl> clone() const override {
+        return make_unique<dynamic_quantize_impl>(*this);
+    }
+
+    void load(BinaryInputBuffer& ib) override {
+        parent::load(ib);
+        if (is_dynamic()) {
+            auto& kernel_selector = kernel_selector_t::Instance();
+            auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName);
+            kernel_impl->GetUpdateDispatchDataFunc(_kernel_data);
+        }
+    }
+
+    static kernel_params_t get_kernel_params(const kernel_impl_params& impl_param, bool is_shape_agnostic = false) {
+        /// TODO: handle group_size here
+        auto params = get_default_params<kernel_selector::dynamic_quantize_params>(impl_param, is_shape_agnostic);
+        params.outputs.push_back(convert_data_tensor(impl_param.get_output_layout(1)));
+
+        return params;
+    }
+
+    void update_dispatch_data(const kernel_impl_params& impl_param) override {
+        auto kernel_params = get_kernel_params(impl_param, true);
+        (_kernel_data.update_dispatch_data_func)(kernel_params, _kernel_data);
+    }
+};
+
+namespace detail {
+
+attach_dynamic_quantize_impl::attach_dynamic_quantize_impl() {
+    auto types = {
+        data_types::f16,
+        data_types::i8
+    };
+
+    auto formats = {
+        format::bfyx,
+    };
+
+    implementation_map<dynamic_quantize>::add(impl_types::ocl,
+                                    shape_types::any,
+                                    typed_primitive_impl_ocl<dynamic_quantize>::create<dynamic_quantize_impl>,
+                                    types,
+                                    formats);
+}
+
+}  // namespace detail
+}  // namespace ocl
+}  // namespace cldnn
+
+BIND_BINARY_BUFFER_WITH_TYPE(cldnn::ocl::dynamic_quantize_impl)
+BIND_BINARY_BUFFER_WITH_TYPE(cldnn::dynamic_quantize)

src/plugins/intel_gpu/src/graph/impls/ocl/register.cpp

@@ -23,6 +23,7 @@ void register_implementations() {
     REGISTER_OCL(depth_to_space);
     REGISTER_OCL(detection_output);
     REGISTER_OCL(dft);
+    REGISTER_OCL(dynamic_quantize);
     REGISTER_OCL(batch_to_space);
     REGISTER_OCL(experimental_detectron_detection_output);
     REGISTER_OCL(experimental_detectron_generate_proposals_single_image);