[CPU] [ARM] FullyConnected: int8 support

eshoguli · Jul 18, 2024 · f62c159 · f62c159
1 parent a58e4a5
commit f62c159
Show file tree

Hide file tree

Showing 106 changed files with 955 additions and 51 deletions.
diff --git a/src/plugins/intel_cpu/src/config.h b/src/plugins/intel_cpu/src/config.h
@@ -44,7 +44,8 @@ struct Config {
         Unknown
     };
 
-    bool collectPerfCounters = false;
+    // TODO: workaround to collect performance counters
+    bool collectPerfCounters = true;
     bool exclusiveAsyncRequests = false;
     SnippetsMode snippetsMode = SnippetsMode::Enable;
     std::string dumpToDot = {};

diff --git a/src/plugins/intel_cpu/src/cpu_memory.cpp b/src/plugins/intel_cpu/src/cpu_memory.cpp
@@ -475,6 +475,7 @@ void DnnlMemoryMngr::notifyUpdate() {
 
 StaticMemory::StaticMemory(const dnnl::engine& eng, MemoryDescPtr desc, const void* data, bool pads_zeroing) :
     m_eng(eng), m_pMemDesc(desc) {
+    OPENVINO_ASSERT(!desc->empty() || (desc->empty() && (data == nullptr)));
     if (desc->getPrecision() == element::string) {
         OPENVINO_THROW("[CPU] StaticMemory object cannot be created for string data.");
     }
@@ -484,7 +485,7 @@ StaticMemory::StaticMemory(const dnnl::engine& eng, MemoryDescPtr desc, const vo
 
     m_size = m_pMemDesc->getCurrentMemSize();
 
-    if (data) {
+    if (data || desc->empty()) {
         m_pMemMngr = std::make_shared<StaticMemoryMngr>(const_cast<void*>(data), m_size);
     } else {
         m_pMemMngr = std::make_shared<StaticMemoryMngr>(m_size);

diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_common_executor.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_common_executor.cpp
@@ -2,6 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+#include "src/core/utils/quantization/AsymmHelpers.h"
+
 #include "acl_common_executor.hpp"
 #include "acl_utils.hpp"
 #include "nodes/executors/memory_arguments.hpp"
@@ -38,9 +40,9 @@ static void initACLTensorParams(const MemoryPtr& memoryPtr,
     }
 }
 
-static ACLInfo initTensorInfo(const arm_compute::TensorShape& tensorShape,
-                              const arm_compute::DataType& dataType,
-                              const arm_compute::DataLayout& dataLayout) {
+ACLInfo ACLCommonExecutor::initTensorInfo(const arm_compute::TensorShape& tensorShape,
+                                          const arm_compute::DataType& dataType,
+                                          const arm_compute::DataLayout& dataLayout) {
     ACLInfo aclMemoryInfo = nullptr;
     if (dataType != arm_compute::DataType::UNKNOWN) {
         aclMemoryInfo = std::make_shared<arm_compute::TensorInfo>(
@@ -66,7 +68,16 @@ bool ACLCommonExecutor::update(const MemoryArgs &memory) {
     ACLMemoryTypes   aclDataType{};
     ACLMemoryLayouts aclDataLayout{};
     for (auto& cpu_mem_ptr : memory) {
+        // TODO: don't init empty tensor
+        if (cpu_mem_ptr.second->getSize() == 0) {
+            continue;
+        }
         const ACLArgs index = argConvert.at(cpu_mem_ptr.first);
+
+        if (index == ACLArgs::ACL_DST) {
+            std::cout << std::endl;
+        }
+
         initACLTensorParams(cpu_mem_ptr.second, aclTensorAttrs,
                             aclMemoryShapes[index],
                             aclDataType[index],
@@ -79,6 +90,9 @@ bool ACLCommonExecutor::update(const MemoryArgs &memory) {
     // Initialize arm_compute::TensorInfo objects
     ACLMemoryInfo aclMemoryInfos;
     for (int i = 0; i < ACLArgs::COUNT_OF_ARGS; i++) {
+        if (i == ACLArgs::ACL_DST) {
+            std::cout << std::endl;
+        }
         aclMemoryInfos[i] = initTensorInfo(aclMemoryShapes[i], aclDataType[i], aclDataLayout[i]);
     }
 
@@ -108,6 +122,7 @@ void ACLCommonExecutor::execute(const MemoryArgs &memory) {
             aclMemoryTensors[index]->allocator()->import_memory(memory.at(cpu_mem_ptr.first)->getData());
         }
     }
+
     iFunction->run();
 }
 

diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_common_executor.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_common_executor.hpp
@@ -52,6 +52,10 @@ class ACLCommonExecutor : public Executor {
 protected:
     ACLTensorAttrs aclTensorAttrs;
 
+    virtual ACLInfo initTensorInfo(const arm_compute::TensorShape& tensorShape,
+                                   const arm_compute::DataType& dataType,
+                                   const arm_compute::DataLayout& dataLayout);
+
 private:
     ACLMemoryTensors aclMemoryTensors;
     ACLFunction iFunction = nullptr;

diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_gemm.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_gemm.cpp
@@ -0,0 +1,92 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "acl_gemm.hpp"
+
+#include "arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h"
+
+#include "nodes/executors/executor.hpp"
+#include "nodes/executors/memory_arguments.hpp"
+#include "utils/debug_capabilities.h"
+#include "nodes/executors/debug_messages.hpp"
+#include "nodes/executors/implementation_utils.hpp"
+
+namespace ov {
+namespace intel_cpu {
+
+ACLGEMMExecutor::ACLGEMMExecutor(const GEMMAttrs &attrs,
+                                 const PostOps &postOps,
+                                 const MemoryArgs &memory,
+                                 const ExecutorContext::CPtr context) {
+    aclTensorAttrs.hasLayoutTypeNHWC = memory.at(ARG_SRC)->getDescPtr()->hasLayoutType(LayoutType::nspc);
+}
+
+bool ACLGEMMExecutor::supports(const GEMMConfig &config) {
+    // TODO: check weights layout
+    const auto attrs = static_cast<GEMMAttrs>(config.attrs);
+    if (std::any_of(
+            attrs.dequantizationScales.begin(),
+            attrs.dequantizationScales.end(),
+            [](float value) { return value != 1.f;})) {
+        return false;
+    }
+
+    const auto src1_dims = std::dynamic_pointer_cast<BlockedMemoryDesc>(config.descs.at(ARG_SRC))->getBlockDims();
+    const auto src2_dims = std::dynamic_pointer_cast<BlockedMemoryDesc>(config.descs.at(ARG_WEI))->getBlockDims();
+
+    VERIFY(one_of(srcType(config), ov::element::f16, ov::element::f32, ov::element::i8, ov::element::u8), UNSUPPORTED_SRC_PRECISIONS);
+    VERIFY(postOpsNumbers(config) < 2,          UNSUPPORTED_NUMBER_OF_POSTOPS);
+    VERIFY(one_of(srcRank(config), 2U, 3U, 4U), UNSUPPORTED_SRC_RANK);
+    VERIFY(one_of(weiRank(config), 2U, 3U, 4U),     UNSUPPORTED_WEI_RANK);
+    VERIFY(static_cast<GEMMAttrs>(config.attrs).dequantizationScales.size() <= 1, UNSUPPORTED_PER_CHANNEL_QUANTIZATION);
+    return true;
+}
+
+void ACLGEMMExecutor::updateTensorsShapes(ACLMemoryShapes& aclMemoryShapes) {}
+
+arm_compute::Status ACLGEMMExecutor::validateTensorsInfo(const ACLMemoryInfo & aclMemoryInfos) {
+    const auto matMulValid = arm_compute::NEGEMMLowpMatrixMultiplyCore::validate(
+            aclMemoryInfos[ACLArgs::ACL_SRC_0].get(),
+            aclMemoryInfos[ACLArgs::ACL_WEI].get(),
+            aclMemoryInfos[ACLArgs::ACL_BIAS].get(),
+            aclMemoryInfos[ACLArgs::ACL_DST].get(),
+            gemmInfo);
+    return matMulValid;
+}
+
+ACLFunction ACLGEMMExecutor::configureFunction(const ACLMemoryTensors & aclMemoryTensors) {
+    auto matMull = std::make_unique<arm_compute::NEGEMMLowpMatrixMultiplyCore>();
+    matMull->configure(
+            aclMemoryTensors[ACLArgs::ACL_SRC_0].get(),
+            aclMemoryTensors[ACLArgs::ACL_WEI].get(),
+            // TODO: fix me
+            nullptr, //aclMemoryTensors[ACLArgs::ACL_BIAS].get(),
+            aclMemoryTensors.at(ACLArgs::ACL_DST).get());
+    return matMull;
+}
+
+ACLInfo ACLGEMMExecutor::initTensorInfo(const arm_compute::TensorShape& tensorShape,
+                                          const arm_compute::DataType& dataType,
+                                          const arm_compute::DataLayout& dataLayout) {
+    arm_compute::DataType fcDataType;
+    switch (dataType) {
+        case arm_compute::DataType::S8: {
+            fcDataType = arm_compute::DataType::QASYMM8_SIGNED;
+            break;
+        }
+        case arm_compute::DataType::U8: {
+            fcDataType = arm_compute::DataType::QASYMM8;
+            break;
+        }
+        default: {
+            fcDataType = dataType;
+            break;
+        }
+    }
+
+    return ACLCommonExecutor::initTensorInfo(tensorShape, fcDataType, dataLayout);
+}
+
+}   // namespace intel_cpu
+}   // namespace ov
diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_gemm.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_gemm.hpp
@@ -0,0 +1,46 @@
+// Copyright (C) 2018-2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "acl_common_executor.hpp"
+#include "nodes/executors/gemm_config.hpp"
+
+namespace ov {
+namespace intel_cpu {
+
+class ACLGEMMExecutor : public ACLCommonExecutor {
+public:
+    ACLGEMMExecutor(const GEMMAttrs& attrs,
+                      const PostOps& postOps,
+                      const MemoryArgs& memory,
+                      const ExecutorContext::CPtr context);
+
+    static bool supports(const GEMMConfig& config);
+
+    void updateTensorsShapes(ACLMemoryShapes& aclMemoryShapes) override;
+
+    arm_compute::Status validateTensorsInfo(const ACLMemoryInfo & aclMemoryInfos) override;
+
+    ACLFunction configureFunction(const ACLMemoryTensors & aclMemoryTensors) override;
+
+    impl_desc_type implType() const override {
+        return impl_desc_type::gemm_acl;
+    }
+
+protected:
+    ACLInfo initTensorInfo(const arm_compute::TensorShape& tensorShape,
+                           const arm_compute::DataType& dataType,
+                           const arm_compute::DataLayout& dataLayout) override;
+
+private:
+    arm_compute::FullyConnectedLayerInfo fullyConnectedLayerInfo;
+    arm_compute::GEMMInfo gemmInfo;
+    arm_compute::WeightsInfo weightsInfo;
+};
+
+using ACLGEMMExecutorPtr = std::shared_ptr<ACLGEMMExecutor>;
+
+}  // namespace intel_cpu
+}  // namespace ov
diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_utils.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_utils.hpp
@@ -101,7 +101,7 @@ inline int axisCast(const std::size_t axis, const std::size_t shapeSize, ACLAxis
 * @param precision precision to be converted
 * @return ComputeLibrary DataType or UNKNOWN if precision is not mapped to DataType
 */
-inline arm_compute::DataType precisionToAclDataType(ov::element::Type precision) {
+inline arm_compute::DataType precisionToAclDataType(const ov::element::Type& precision) {
     switch (precision) {
         case ov::element::i8:    return arm_compute::DataType::S8;
         case ov::element::u8:    return arm_compute::DataType::U8;

diff --git a/src/plugins/intel_cpu/src/nodes/executors/debug_messages.hpp b/src/plugins/intel_cpu/src/nodes/executors/debug_messages.hpp
@@ -17,6 +17,7 @@
 #define UNSUPPORTED_DST_RANK " unsupported dst rank"
 #define UNSUPPORTED_DST_STRIDES " unsupported dst strides"
 #define HEURISTICS_MISMATCH " heuristics mismatch"
+#define UNSUPPORTED_PER_CHANNEL_QUANTIZATION " unsupported per-channel quantization"
 
 #define VERIFY(condition, ...) \
     do { \

diff --git a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_config.hpp b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_config.hpp
@@ -17,7 +17,8 @@ struct FCAttrs {
     // @todo probably we don't want with bias flag, since this information is already
     // a part of src memory descs
     bool withBias = false;
-    bool weightsNonTransposed = false;
+    // TODO: why default is false???
+    bool weightsNonTransposed = true;
     bool sparseWeights = false;
     // @todo only memory descriptors should be a part of attributes
     // actual memory should be passed into "execute" or "prepareMemory" calls

diff --git a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp
@@ -77,7 +77,7 @@ static const TypeMapping dnnlFCTypeMapping {
 
 static const TypeMapping aclFCTypeMapping {
     // {src, wei, bia, dst}              pt<src, wei, bias, dst>
-    {{_f32 | _f16, _any, _any, _any}, pt(bypass(), use<0>(), use<0>(), use<0>())},
+    {{_i8, _i8, _any, _any},          pt(just<i8>(), just<i8>(), just<i32>(), just<i32>())},
     {{_any, _any, _any, _any},        pt(just<f32>(), just<f32>(), just<f32>(), just<f32>())}
 };
 

diff --git a/src/plugins/intel_cpu/src/nodes/executors/gemm_attrs.hpp b/src/plugins/intel_cpu/src/nodes/executors/gemm_attrs.hpp
@@ -0,0 +1,34 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <vector>
+
+#include "cpu_memory.h"
+#include "executor_config.hpp"
+
+namespace ov {
+namespace intel_cpu {
+
+// @todo require explicit initialization of all the attributes?
+struct GEMMAttrs {
+    // @todo probably we don't want with bias flag, since this information is already
+    // a part of src memory descs
+    bool withBias = false;
+    // TODO: why default is false???
+    bool weightsNonTransposed = true;
+    bool sparseWeights = false;
+    // @todo only memory descriptors should be a part of attributes
+    // actual memory should be passed into "execute" or "prepareMemory" calls
+    std::vector<float> dequantizationScales;
+    // @todo should be passed as an additional memory input?
+    MemoryCPtr decompressionSubtractPtr;
+    MemoryCPtr decompressionMultiplyPtr;
+    uint64_t dynamicQuantizationGroupSize;
+    ov::intel_cpu::Config::ModelType modelType = ov::intel_cpu::Config::ModelType::Unknown;
+};
+
+}  // namespace intel_cpu
+}  // namespace ov
diff --git a/src/plugins/intel_cpu/src/nodes/executors/gemm_config.hpp b/src/plugins/intel_cpu/src/nodes/executors/gemm_config.hpp
@@ -0,0 +1,14 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "executor_config.hpp"
+#include "gemm_attrs.hpp"
+
+namespace ov {
+namespace intel_cpu {
+using GEMMConfig = ov::intel_cpu::executor::Config<GEMMAttrs>;
+}  // namespace intel_cpu
+}  // namespace ov