Skip to content

Commit

Permalink
POC for supporting blob metadata to be read from std::stream instead
Browse files Browse the repository at this point in the history 
of `std::vector<uint8_t>`
  • Loading branch information
@MirceaDan99
MirceaDan99 committed Dec 15, 2024
1 parent 7bfc852 commit ee210c9
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 92 deletions.
14 changes: 6 additions & 8 deletions src/plugins/intel_npu/src/common/include/intel_npu/common/blob_container.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,29 +47,27 @@ class BlobContainerVector : public BlobContainer {
class BlobContainerAlignedBuffer : public BlobContainer {
public:
BlobContainerAlignedBuffer(const std::shared_ptr<ov::AlignedBuffer>& blobSO,
size_t ovHeaderOffset,
size_t metadataSize)
: _ownershipBlob(blobSO),
size_t ovHeaderOffset, uint64_t blobSize)
: _blobSize(blobSize),
_ovHeaderOffset(ovHeaderOffset),
_metadataSize(metadataSize) {}
_ownershipBlob(blobSO) {}

void* get_ptr() override {
return _ownershipBlob->get_ptr(_ovHeaderOffset);
}

size_t size() const override {
// remove OV header offset and metadata from blob size
return _ownershipBlob->size() - _ovHeaderOffset - _metadataSize;
return _blobSize;
}

bool release_from_memory() override {
return false;
}

private:
std::shared_ptr<ov::AlignedBuffer> _ownershipBlob;
uint64_t _blobSize;
size_t _ovHeaderOffset;
size_t _metadataSize;
std::shared_ptr<ov::AlignedBuffer> _ownershipBlob;
};

} // namespace intel_npu
4 changes: 2 additions & 2 deletions src/plugins/intel_npu/src/compiler_adapter/src/plugin_graph.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ PluginGraph::PluginGraph(const std::shared_ptr<ZeGraphExtWrappers>& zeGraphExt,
}

size_t PluginGraph::export_blob(std::ostream& stream) const {
stream.write(reinterpret_cast<const char*>(_blob.data()), _blob.size());
stream.write(reinterpret_cast<const char*>(_blobPtr->get_ptr()), _blobPtr->size());

if (!stream) {
_logger.error("Write blob to stream failed. Blob is broken!");
Expand All @@ -51,7 +51,7 @@ size_t PluginGraph::export_blob(std::ostream& stream) const {
_logger.info(str.str().c_str());
}
_logger.info("Write blob to stream successfully.");
return _blob.size();
return _blobPtr->size();
}

std::vector<ov::ProfilingInfo> PluginGraph::process_profiling_output(const std::vector<uint8_t>& profData,
Expand Down
22 changes: 19 additions & 3 deletions src/plugins/intel_npu/src/plugin/include/metadata.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
#include <string>
#include <vector>

#include <openvino/runtime/aligned_buffer.hpp>

namespace intel_npu {

/**
Expand Down Expand Up @@ -98,6 +100,10 @@ struct MetadataBase {

virtual bool is_compatible() = 0;

virtual uint64_t get_blob_size() const = 0;

virtual size_t get_ov_header_offset() const = 0;

virtual ~MetadataBase() = default;
};

Expand All @@ -114,13 +120,17 @@ struct Metadata : public MetadataBase {};
*/
template <>
struct Metadata<METADATA_VERSION_1_0> : public MetadataBase {
protected:
private:
uint64_t _blobDataSize;
uint32_t _version;
size_t _ovHeaderOffset;
OpenvinoVersion _ovVersion;

public:
Metadata();

Metadata(size_t ovHeaderOffset, uint64_t blobDataSize);

void read(std::istream& stream) override;

void write(std::ostream& stream) override;
Expand All @@ -142,6 +152,10 @@ struct Metadata<METADATA_VERSION_1_0> : public MetadataBase {
void set_version(uint32_t newVersion);

void set_ov_version(const OpenvinoVersion& newVersion);

uint64_t get_blob_size() const override;

size_t get_ov_header_offset() const override;
};

/**
Expand All @@ -150,14 +164,16 @@ struct Metadata<METADATA_VERSION_1_0> : public MetadataBase {
* @return Unique pointer to the created MetadataBase object if the major version is supported; otherwise, returns
* 'nullptr'.
*/
std::unique_ptr<MetadataBase> create_metadata(uint32_t version);
std::unique_ptr<MetadataBase> create_metadata(uint32_t version, size_t ovHeaderOffset, uint64_t blobDataSize);

/**
* @brief Reads metadata from a blob.
*
* @return If the blob is versioned and its major version is supported, returns an unique pointer to the read
* MetadataBase object; otherwise, returns 'nullptr'.
*/
std::unique_ptr<MetadataBase> read_metadata_from(const std::vector<uint8_t>& blob);
std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream);

std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream, const std::shared_ptr<ov::AlignedBuffer>& modelBuffer);

} // namespace intel_npu
122 changes: 101 additions & 21 deletions src/plugins/intel_npu/src/plugin/src/metadata.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,35 @@
#include "intel_npu/config/config.hpp"
#include "intel_npu/utils/logger/logger.hpp"
#include "openvino/core/version.hpp"
#include "openvino/runtime/shared_buffer.hpp"

namespace {

size_t getFileSize(std::istream& stream) {
auto log = intel_npu::Logger::global().clone("getFileSize");
if (!stream) {
OPENVINO_THROW("Stream is in bad status! Please check the passed stream status!");
}

const size_t streamStart = stream.tellg();
stream.seekg(0, std::ios_base::end);
const size_t streamEnd = stream.tellg();
stream.seekg(streamStart, std::ios_base::beg);

log.debug("Read blob size: streamStart=%zu, streamEnd=%zu", streamStart, streamEnd);

if (streamEnd < streamStart) {
OPENVINO_THROW("Invalid stream size: streamEnd (",
streamEnd,
") is not larger than streamStart (",
streamStart,
")!");
}

return streamEnd - streamStart;
}

} // namespace

namespace intel_npu {

Expand All @@ -24,7 +53,15 @@ void OpenvinoVersion::read(std::istream& stream) {
}

Metadata<METADATA_VERSION_1_0>::Metadata()
: _version{METADATA_VERSION_1_0},
: _blobDataSize{0},
_version{METADATA_VERSION_1_0},
_ovHeaderOffset{0},
_ovVersion{ov::get_openvino_version().buildNumber} {}

Metadata<METADATA_VERSION_1_0>::Metadata(size_t ovHeaderOffset, uint64_t blobDataSize)
: _blobDataSize{blobDataSize},
_version{METADATA_VERSION_1_0},
_ovHeaderOffset{ovHeaderOffset},
_ovVersion{ov::get_openvino_version().buildNumber} {}

void Metadata<METADATA_VERSION_1_0>::read(std::istream& stream) {
Expand All @@ -41,13 +78,13 @@ void Metadata<METADATA_VERSION_1_0>::write(std::ostream& stream) {
stream.write(_ovVersion.get_version().data(), _ovVersion.get_version().size());
}

std::unique_ptr<MetadataBase> create_metadata(uint32_t version) {
std::unique_ptr<MetadataBase> create_metadata(uint32_t version, size_t ovHeaderOffset, uint64_t blobDataSize) {
switch (version) {
case METADATA_VERSION_1_0:
return std::make_unique<Metadata<METADATA_VERSION_1_0>>();
return std::make_unique<Metadata<METADATA_VERSION_1_0>>(ovHeaderOffset, blobDataSize);

default:
OPENVINO_THROW("Invalid metadata version!");
return nullptr;
}
}

Expand Down Expand Up @@ -75,35 +112,70 @@ bool Metadata<METADATA_VERSION_1_0>::is_compatible() {
return true;
}

std::unique_ptr<MetadataBase> read_metadata_from(const std::vector<uint8_t>& blob) {
std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream) {
Logger logger("NPUPlugin", Logger::global().level());
size_t magicBytesSize = MAGIC_BYTES.size();
std::string blobMagicBytes;
blobMagicBytes.resize(magicBytesSize);

auto metadataIterator = blob.end() - magicBytesSize;
std::memcpy(blobMagicBytes.data(), &(*metadataIterator), magicBytesSize);
size_t currentStreamPos = stream.tellg();
size_t streamSize = getFileSize(stream);
stream.seekg(streamSize - magicBytesSize, std::ios::beg);
stream.read(blobMagicBytes.data(), magicBytesSize);
if (MAGIC_BYTES != blobMagicBytes) {
OPENVINO_THROW("Blob is missing NPU metadata!");
logger.error("Blob is missing NPU metadata!");
return nullptr;
}

uint64_t blobDataSize;
metadataIterator -= sizeof(blobDataSize);
std::memcpy(&blobDataSize, &(*metadataIterator), sizeof(blobDataSize));
metadataIterator = blob.begin() + blobDataSize;

std::stringstream metadataStream;
metadataStream.write(reinterpret_cast<const char*>(&(*metadataIterator)),
blob.end() - metadataIterator - sizeof(blobDataSize));
stream.seekg(streamSize - magicBytesSize - sizeof(blobDataSize), std::ios::beg);
stream.read(reinterpret_cast<char*>(&blobDataSize), sizeof(blobDataSize));
stream.seekg(currentStreamPos + blobDataSize, std::ios::beg);

uint32_t metaVersion;
metadataStream.read(reinterpret_cast<char*>(&metaVersion), sizeof(metaVersion));
stream.read(reinterpret_cast<char*>(&metaVersion), sizeof(metaVersion));

auto storedMeta = create_metadata(metaVersion, currentStreamPos, blobDataSize);
if (storedMeta != nullptr) {
storedMeta->read(stream);
} else {
logger.warning("Imported blob metadata version: %d.%d, but the current version is: %d.%d",
get_major(metaVersion),
get_minor(metaVersion),
get_major(CURRENT_METADATA_VERSION),
get_minor(CURRENT_METADATA_VERSION));
}
stream.seekg(currentStreamPos, std::ios::beg);
return storedMeta;
}

std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream, const std::shared_ptr<ov::AlignedBuffer>& modelBuffer) {
Logger logger("NPUPlugin", Logger::global().level());
size_t magicBytesSize = MAGIC_BYTES.size();
std::string blobMagicBytes;
blobMagicBytes.resize(magicBytesSize);

std::unique_ptr<MetadataBase> storedMeta;
try {
storedMeta = create_metadata(metaVersion);
storedMeta->read(metadataStream);
} catch(...) {
size_t currentStreamPos = stream.tellg();
size_t streamSize = modelBuffer->size();

blobMagicBytes.assign(reinterpret_cast<const char*>(modelBuffer->get_ptr(streamSize - magicBytesSize)), magicBytesSize);
if (MAGIC_BYTES != blobMagicBytes) {
logger.error("Blob is missing NPU metadata!");
return nullptr;
}

uint64_t blobDataSize;
blobDataSize = *reinterpret_cast<uint64_t*>(modelBuffer->get_ptr(streamSize - magicBytesSize - sizeof(blobDataSize)));

uint32_t metaVersion;
metaVersion = *reinterpret_cast<uint32_t*>(modelBuffer->get_ptr(currentStreamPos + blobDataSize));

auto storedMeta = create_metadata(metaVersion, currentStreamPos, blobDataSize);
stream.seekg(blobDataSize + sizeof(metaVersion), std::ios::cur);
if (storedMeta != nullptr) {
storedMeta->read(stream);
} else {
>>>>>>> a76d0bdf91 (POC for supporting blob metadata)
logger.warning("Imported blob metadata version: %d.%d, but the current version is: %d.%d",
get_major(metaVersion),
get_minor(metaVersion),
Expand All @@ -121,4 +193,12 @@ void Metadata<METADATA_VERSION_1_0>::set_ov_version(const OpenvinoVersion& newVe
_ovVersion = newVersion;
}

uint64_t Metadata<METADATA_VERSION_1_0>::get_blob_size() const {
return _blobDataSize;
}

size_t Metadata<METADATA_VERSION_1_0>::get_ov_header_offset() const {
return _ovHeaderOffset;
}

} // namespace intel_npu
42 changes: 15 additions & 27 deletions src/plugins/intel_npu/src/plugin/src/plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,30 +132,6 @@ std::map<std::string, std::string> any_copy(const ov::AnyMap& params) {
return result;
}

size_t getFileSize(std::istream& stream) {
auto log = Logger::global().clone("getFileSize");
if (!stream) {
OPENVINO_THROW("Stream is in bad status! Please check the passed stream status!");
}

const size_t streamStart = stream.tellg();
stream.seekg(0, std::ios_base::end);
const size_t streamEnd = stream.tellg();
stream.seekg(streamStart, std::ios_base::beg);

log.debug("Read blob size: streamStart=%zu, streamEnd=%zu", streamStart, streamEnd);

if (streamEnd < streamStart) {
OPENVINO_THROW("Invalid stream size: streamEnd (",
streamEnd,
") is not larger than streamStart (",
streamStart,
")!");
}

return streamEnd - streamStart;
}

void update_log_level(const std::map<std::string, std::string>& propertiesMap) {
auto it = propertiesMap.find(std::string(LOG_LEVEL::key()));
if (it != propertiesMap.end()) {
Expand Down Expand Up @@ -784,10 +760,22 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, c
auto compiler = compilerAdapterFactory.getCompiler(_backends->getIEngineBackend(), localConfig);

std::unique_ptr<BlobContainer> blobPtr;
std::unique_ptr<MetadataBase> storedMeta;

if (modelBuffer == nullptr) {
auto graphSize = getFileSize(stream);
if (dynamic_cast<ov::OwningSharedStreamBuffer*>(stream.rdbuf())) {
storedMeta = read_metadata_from(stream, dynamic_cast<ov::OwningSharedStreamBuffer*>(stream.rdbuf())->get_buffer());
} else {
storedMeta = read_metadata_from(stream);
}

if (storedMeta == nullptr) {
OPENVINO_THROW("Could not read metadata!");
} else if (!storedMeta->is_compatible()) {
OPENVINO_THROW("Incompatible blob version!");
}
auto graphSize = storedMeta->get_blob_size();

if (modelBuffer == nullptr) {
std::vector<uint8_t> blob(graphSize);
stream.read(reinterpret_cast<char*>(blob.data()), graphSize);
if (!stream) {
Expand All @@ -797,7 +785,7 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, c

blobPtr = std::make_unique<BlobContainerVector>(std::move(blob));
} else {
blobPtr = std::make_unique<BlobContainerAlignedBuffer>(modelBuffer, stream.tellg(), 0);
blobPtr = std::make_unique<BlobContainerAlignedBuffer>(modelBuffer, storedMeta->get_ov_header_offset(), graphSize);
}

auto storedMeta = read_metadata_from(blob);
Expand Down
Loading

0 comments on commit ee210c9

Please sign in to comment.