Skip to content

Commit

Permalink
MKLDNNMemory::SetData extended with data conversion for unsupported b…
Browse files Browse the repository at this point in the history 
…y reorders cases.
  • Loading branch information
@maxnick
maxnick committed Dec 28, 2020
1 parent a23a2cc commit d2fdb9f
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 45 deletions.
107 changes: 62 additions & 45 deletions inference-engine/src/mkldnn_plugin/mkldnn_memory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "mkldnn_node.h"
#include "mkldnn_extension_utils.h"
#include "nodes/common/cpu_memcpy.h"
#include "nodes/common/cpu_convert.h"

using namespace InferenceEngine;
using namespace mkldnn;
Expand Down Expand Up @@ -90,44 +91,64 @@ void MKLDNNMemory::Create(const mkldnn::memory::desc& desc, const void *data, bo
}

void MKLDNNMemory::SetData(memory::data_type dataType, memory::format format, const void* data, size_t size, bool ftz) const {
uint8_t itemSize = MKLDNNExtensionUtils::sizeOfDataType(mkldnn::memory::data_type(dataType));

if (static_cast<mkldnn_memory_format_t>(format) != GetDescriptor().data.format ||
GetDataType() != dataType) {
auto memData = GetDescriptor().data;

std::vector<ptrdiff_t> dims(memData.dims, memData.dims + memData.ndims);

MKLDNNMemory src(eng);
src.Create(dims, dataType, format, data);

std::shared_ptr<mkldnn::reorder> pReorder =
std::shared_ptr<mkldnn::reorder>(new mkldnn::reorder(src.GetPrimitive(), GetPrimitive()));

mkldnn::stream(stream::kind::eager).submit({*pReorder});
} else {
if (GetDataType() == dataType && static_cast<mkldnn_memory_format_t>(format) == GetDescriptor().data.format) {
uint8_t itemSize = MKLDNNExtensionUtils::sizeOfDataType(mkldnn::memory::data_type(dataType));
uint8_t* dataPtr = static_cast<uint8_t*>(GetData());
// We cannot support strides for i/o blobs because it affects performance.
dataPtr += itemSize * prim->get_primitive_desc().desc().data.layout_desc.blocking.offset_padding;
cpu_memcpy(dataPtr, data, size);
}

if (ftz && dataType == mkldnn_f32) {
// Internal blobs haven't strides yet.
auto *memData = static_cast<float *>(GetData());
memData += prim->get_primitive_desc().desc().data.layout_desc.blocking.offset_padding;
size_t realSize = GetSize() / sizeof(float);
for (size_t i = 0; i < realSize; i++) {
if (memData[i] != 0 && (fabsf(memData[i]) < std::numeric_limits<float>::min())) {
memData[i] = 0.0f;
if (ftz && dataType == mkldnn_f32) {
// Internal blobs haven't strides yet.
auto *memData = static_cast<float *>(GetData());
memData += prim->get_primitive_desc().desc().data.layout_desc.blocking.offset_padding;
size_t realSize = GetSize() / sizeof(float);
for (size_t i = 0; i < realSize; i++) {
if (memData[i] != 0 && (fabsf(memData[i]) < std::numeric_limits<float>::min())) {
memData[i] = 0.0f;
}
}
}
} else {
auto memData = this->GetDescriptor().data;
std::vector<ptrdiff_t> dims(memData.dims, memData.dims + memData.ndims);

MKLDNNMemory src(this->eng);
src.Create(dims, dataType, format, data);

this->SetData(src, ftz);
}
}

void MKLDNNMemory::SetData(const MKLDNNMemory& memory, bool ftz) const {
mkldnn::reorder reorderPrim(memory.GetPrimitive(), GetPrimitive());
mkldnn::stream(stream::kind::eager).submit({reorderPrim});
std::unique_ptr<mkldnn::reorder> pReorder;

try {
pReorder = std::unique_ptr<mkldnn::reorder>(new mkldnn::reorder(memory.GetPrimitive(), this->GetPrimitive()));
}
catch (const mkldnn::error& err) {
if (mkldnn_unimplemented == err.status && this->GetDataType() != memory.GetDataType()) {
//we probably could not make the reorder because there is no one supporting this precision conversion
//lets try to convert data first using cpu_convert
std::vector<uint8_t> tmpBuff(memory.GetSize());
auto data = static_cast<const uint8_t*>(memory.GetData()) + memory.GetDescriptor().data.layout_desc.blocking.offset_padding;

cpu_convert(data, tmpBuff.data(), MKLDNNMemoryDesc::convertPrecision(mkldnn_data_type_t(memory.GetDataType())),
MKLDNNMemoryDesc::convertPrecision(mkldnn_data_type_t(this->GetDataType())), memory.GetElementsCount());

MKLDNNMemory src(this->eng);
src.Create(memory.GetDims(), memory.GetDataType(), memory.GetFormat(), data);

pReorder = std::unique_ptr<mkldnn::reorder>(new mkldnn::reorder(src.GetPrimitive(), this->GetPrimitive()));
} else {
throw;
}
}
if (pReorder) {
mkldnn::stream(stream::kind::eager).submit({*pReorder});
} else {
THROW_IE_EXCEPTION << "Could not make mkldnn reorder.";
}

if (ftz && memory.GetDataType() == mkldnn::memory::f32 && GetFormat() != mkldnn::memory::wino_fmt &&
GetDataType() != mkldnn::memory::bf16) {
Expand Down Expand Up @@ -523,36 +544,33 @@ MKLDNNMemoryDesc::MKLDNNMemoryDesc(mkldnn::memory::dims dims, mkldnn::memory::da
MKLDNNMemory::CreateBlockingDesc(desc);
}

MKLDNNMemoryDesc::operator InferenceEngine::TensorDesc() const {
Precision precision;
switch (desc.data.data_type) {

InferenceEngine::Precision MKLDNNMemoryDesc::convertPrecision(mkldnn_data_type_t dataType) {
switch (dataType) {
case mkldnn_f32:
precision = Precision::FP32;
break;
return Precision::FP32;
case mkldnn_u8:
precision = Precision::U8;
break;
return Precision::U8;
case mkldnn_s8:
precision = Precision::I8;
break;
return Precision::I8;
case mkldnn_s16:
precision = Precision::I16;
break;
return Precision::I16;
case mkldnn_s32:
precision = Precision::I32;
break;
return Precision::I32;
case mkldnn_bin:
precision = Precision::BIN;
break;
return Precision::BIN;
case mkldnn_bf16:
precision = Precision::BF16;
break;
return Precision::BF16;
default:
THROW_IE_EXCEPTION << "Cannot cast to TensorDesc. Unsupported precision!";
}
}

MKLDNNMemoryDesc::operator InferenceEngine::TensorDesc() const {
Layout layout;
SizeVector order;
SizeVector blkDims;
Precision precision = convertPrecision(desc.data.data_type);
auto blkInfo = desc.data.layout_desc.blocking;
auto offset = static_cast<size_t>(blkInfo.offset_padding);
SizeVector offsetsForDims;
Expand Down Expand Up @@ -1411,5 +1429,4 @@ bool MKLDNNMemoryDesc::blocksExtended() const {
}
return false;
}

} // namespace MKLDNNPlugin
2 changes: 2 additions & 0 deletions inference-engine/src/mkldnn_plugin/mkldnn_memory.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ class MKLDNNMemoryDesc {
operator mkldnn::memory::desc() const;
operator InferenceEngine::TensorDesc() const;

static InferenceEngine::Precision convertPrecision(mkldnn_data_type_t dataType);

private:
mkldnn::memory::desc desc;
};
Expand Down

0 comments on commit d2fdb9f

Please sign in to comment.