diff --git a/src/plugins/intel_npu/src/common/include/intel_npu/common/blob_container.hpp b/src/plugins/intel_npu/src/common/include/intel_npu/common/blob_container.hpp index 13f833ddfccc88..ed5bb92f473002 100644 --- a/src/plugins/intel_npu/src/common/include/intel_npu/common/blob_container.hpp +++ b/src/plugins/intel_npu/src/common/include/intel_npu/common/blob_container.hpp @@ -47,19 +47,17 @@ class BlobContainerVector : public BlobContainer { class BlobContainerAlignedBuffer : public BlobContainer { public: BlobContainerAlignedBuffer(const std::shared_ptr& blobSO, - size_t ovHeaderOffset, - size_t metadataSize) - : _ownershipBlob(blobSO), + size_t ovHeaderOffset, uint64_t blobSize) + : _blobSize(blobSize), _ovHeaderOffset(ovHeaderOffset), - _metadataSize(metadataSize) {} + _ownershipBlob(blobSO) {} void* get_ptr() override { return _ownershipBlob->get_ptr(_ovHeaderOffset); } size_t size() const override { - // remove OV header offset and metadata from blob size - return _ownershipBlob->size() - _ovHeaderOffset - _metadataSize; + return _blobSize; } bool release_from_memory() override { @@ -67,9 +65,9 @@ class BlobContainerAlignedBuffer : public BlobContainer { } private: - std::shared_ptr _ownershipBlob; + uint64_t _blobSize; size_t _ovHeaderOffset; - size_t _metadataSize; + std::shared_ptr _ownershipBlob; }; } // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler_adapter/src/plugin_graph.cpp b/src/plugins/intel_npu/src/compiler_adapter/src/plugin_graph.cpp index 150ea2e5b72e37..90e531d5b940c7 100644 --- a/src/plugins/intel_npu/src/compiler_adapter/src/plugin_graph.cpp +++ b/src/plugins/intel_npu/src/compiler_adapter/src/plugin_graph.cpp @@ -31,7 +31,7 @@ PluginGraph::PluginGraph(const std::shared_ptr& zeGraphExt, } size_t PluginGraph::export_blob(std::ostream& stream) const { - stream.write(reinterpret_cast(_blob.data()), _blob.size()); + stream.write(reinterpret_cast(_blobPtr->get_ptr()), _blobPtr->size()); if (!stream) { _logger.error("Write blob to stream failed. Blob is broken!"); @@ -51,7 +51,7 @@ size_t PluginGraph::export_blob(std::ostream& stream) const { _logger.info(str.str().c_str()); } _logger.info("Write blob to stream successfully."); - return _blob.size(); + return _blobPtr->size(); } std::vector PluginGraph::process_profiling_output(const std::vector& profData, diff --git a/src/plugins/intel_npu/src/plugin/include/metadata.hpp b/src/plugins/intel_npu/src/plugin/include/metadata.hpp index d91b2a6550de7f..05e45f1c2436ee 100644 --- a/src/plugins/intel_npu/src/plugin/include/metadata.hpp +++ b/src/plugins/intel_npu/src/plugin/include/metadata.hpp @@ -10,6 +10,8 @@ #include #include +#include + namespace intel_npu { /** @@ -98,6 +100,10 @@ struct MetadataBase { virtual bool is_compatible() = 0; + virtual uint64_t get_blob_size() const = 0; + + virtual size_t get_ov_header_offset() const = 0; + virtual ~MetadataBase() = default; }; @@ -114,13 +120,17 @@ struct Metadata : public MetadataBase {}; */ template <> struct Metadata : public MetadataBase { -protected: +private: + uint64_t _blobDataSize; uint32_t _version; + size_t _ovHeaderOffset; OpenvinoVersion _ovVersion; public: Metadata(); + Metadata(size_t ovHeaderOffset, uint64_t blobDataSize); + void read(std::istream& stream) override; void write(std::ostream& stream) override; @@ -142,6 +152,10 @@ struct Metadata : public MetadataBase { void set_version(uint32_t newVersion); void set_ov_version(const OpenvinoVersion& newVersion); + + uint64_t get_blob_size() const override; + + size_t get_ov_header_offset() const override; }; /** @@ -150,7 +164,7 @@ struct Metadata : public MetadataBase { * @return Unique pointer to the created MetadataBase object if the major version is supported; otherwise, returns * 'nullptr'. */ -std::unique_ptr create_metadata(uint32_t version); +std::unique_ptr create_metadata(uint32_t version, size_t ovHeaderOffset, uint64_t blobDataSize); /** * @brief Reads metadata from a blob. @@ -158,6 +172,8 @@ std::unique_ptr create_metadata(uint32_t version); * @return If the blob is versioned and its major version is supported, returns an unique pointer to the read * MetadataBase object; otherwise, returns 'nullptr'. */ -std::unique_ptr read_metadata_from(const std::vector& blob); +std::unique_ptr read_metadata_from(std::istream& stream); + +std::unique_ptr read_metadata_from(std::istream& stream, const std::shared_ptr& modelBuffer); } // namespace intel_npu diff --git a/src/plugins/intel_npu/src/plugin/src/metadata.cpp b/src/plugins/intel_npu/src/plugin/src/metadata.cpp index 175df59b42e713..72322b912f95d7 100644 --- a/src/plugins/intel_npu/src/plugin/src/metadata.cpp +++ b/src/plugins/intel_npu/src/plugin/src/metadata.cpp @@ -10,6 +10,35 @@ #include "intel_npu/config/config.hpp" #include "intel_npu/utils/logger/logger.hpp" #include "openvino/core/version.hpp" +#include "openvino/runtime/shared_buffer.hpp" + +namespace { + +size_t getFileSize(std::istream& stream) { + auto log = intel_npu::Logger::global().clone("getFileSize"); + if (!stream) { + OPENVINO_THROW("Stream is in bad status! Please check the passed stream status!"); + } + + const size_t streamStart = stream.tellg(); + stream.seekg(0, std::ios_base::end); + const size_t streamEnd = stream.tellg(); + stream.seekg(streamStart, std::ios_base::beg); + + log.debug("Read blob size: streamStart=%zu, streamEnd=%zu", streamStart, streamEnd); + + if (streamEnd < streamStart) { + OPENVINO_THROW("Invalid stream size: streamEnd (", + streamEnd, + ") is not larger than streamStart (", + streamStart, + ")!"); + } + + return streamEnd - streamStart; +} + +} // namespace namespace intel_npu { @@ -24,7 +53,15 @@ void OpenvinoVersion::read(std::istream& stream) { } Metadata::Metadata() - : _version{METADATA_VERSION_1_0}, + : _blobDataSize{0}, + _version{METADATA_VERSION_1_0}, + _ovHeaderOffset{0}, + _ovVersion{ov::get_openvino_version().buildNumber} {} + +Metadata::Metadata(size_t ovHeaderOffset, uint64_t blobDataSize) + : _blobDataSize{blobDataSize}, + _version{METADATA_VERSION_1_0}, + _ovHeaderOffset{ovHeaderOffset}, _ovVersion{ov::get_openvino_version().buildNumber} {} void Metadata::read(std::istream& stream) { @@ -41,13 +78,13 @@ void Metadata::write(std::ostream& stream) { stream.write(_ovVersion.get_version().data(), _ovVersion.get_version().size()); } -std::unique_ptr create_metadata(uint32_t version) { +std::unique_ptr create_metadata(uint32_t version, size_t ovHeaderOffset, uint64_t blobDataSize) { switch (version) { case METADATA_VERSION_1_0: - return std::make_unique>(); + return std::make_unique>(ovHeaderOffset, blobDataSize); default: - OPENVINO_THROW("Invalid metadata version!"); + return nullptr; } } @@ -75,35 +112,70 @@ bool Metadata::is_compatible() { return true; } -std::unique_ptr read_metadata_from(const std::vector& blob) { +std::unique_ptr read_metadata_from(std::istream& stream) { Logger logger("NPUPlugin", Logger::global().level()); size_t magicBytesSize = MAGIC_BYTES.size(); std::string blobMagicBytes; blobMagicBytes.resize(magicBytesSize); - auto metadataIterator = blob.end() - magicBytesSize; - std::memcpy(blobMagicBytes.data(), &(*metadataIterator), magicBytesSize); + size_t currentStreamPos = stream.tellg(); + size_t streamSize = getFileSize(stream); + stream.seekg(streamSize - magicBytesSize, std::ios::beg); + stream.read(blobMagicBytes.data(), magicBytesSize); if (MAGIC_BYTES != blobMagicBytes) { - OPENVINO_THROW("Blob is missing NPU metadata!"); + logger.error("Blob is missing NPU metadata!"); + return nullptr; } uint64_t blobDataSize; - metadataIterator -= sizeof(blobDataSize); - std::memcpy(&blobDataSize, &(*metadataIterator), sizeof(blobDataSize)); - metadataIterator = blob.begin() + blobDataSize; - - std::stringstream metadataStream; - metadataStream.write(reinterpret_cast(&(*metadataIterator)), - blob.end() - metadataIterator - sizeof(blobDataSize)); + stream.seekg(streamSize - magicBytesSize - sizeof(blobDataSize), std::ios::beg); + stream.read(reinterpret_cast(&blobDataSize), sizeof(blobDataSize)); + stream.seekg(currentStreamPos + blobDataSize, std::ios::beg); uint32_t metaVersion; - metadataStream.read(reinterpret_cast(&metaVersion), sizeof(metaVersion)); + stream.read(reinterpret_cast(&metaVersion), sizeof(metaVersion)); + + auto storedMeta = create_metadata(metaVersion, currentStreamPos, blobDataSize); + if (storedMeta != nullptr) { + storedMeta->read(stream); + } else { + logger.warning("Imported blob metadata version: %d.%d, but the current version is: %d.%d", + get_major(metaVersion), + get_minor(metaVersion), + get_major(CURRENT_METADATA_VERSION), + get_minor(CURRENT_METADATA_VERSION)); + } + stream.seekg(currentStreamPos, std::ios::beg); + return storedMeta; +} + +std::unique_ptr read_metadata_from(std::istream& stream, const std::shared_ptr& modelBuffer) { + Logger logger("NPUPlugin", Logger::global().level()); + size_t magicBytesSize = MAGIC_BYTES.size(); + std::string blobMagicBytes; + blobMagicBytes.resize(magicBytesSize); - std::unique_ptr storedMeta; - try { - storedMeta = create_metadata(metaVersion); - storedMeta->read(metadataStream); - } catch(...) { + size_t currentStreamPos = stream.tellg(); + size_t streamSize = modelBuffer->size(); + + blobMagicBytes.assign(reinterpret_cast(modelBuffer->get_ptr(streamSize - magicBytesSize)), magicBytesSize); + if (MAGIC_BYTES != blobMagicBytes) { + logger.error("Blob is missing NPU metadata!"); + return nullptr; + } + + uint64_t blobDataSize; + blobDataSize = *reinterpret_cast(modelBuffer->get_ptr(streamSize - magicBytesSize - sizeof(blobDataSize))); + + uint32_t metaVersion; + metaVersion = *reinterpret_cast(modelBuffer->get_ptr(currentStreamPos + blobDataSize)); + + auto storedMeta = create_metadata(metaVersion, currentStreamPos, blobDataSize); + stream.seekg(blobDataSize + sizeof(metaVersion), std::ios::cur); + if (storedMeta != nullptr) { + storedMeta->read(stream); + } else { +>>>>>>> a76d0bdf91 (POC for supporting blob metadata) logger.warning("Imported blob metadata version: %d.%d, but the current version is: %d.%d", get_major(metaVersion), get_minor(metaVersion), @@ -121,4 +193,12 @@ void Metadata::set_ov_version(const OpenvinoVersion& newVe _ovVersion = newVersion; } +uint64_t Metadata::get_blob_size() const { + return _blobDataSize; +} + +size_t Metadata::get_ov_header_offset() const { + return _ovHeaderOffset; +} + } // namespace intel_npu diff --git a/src/plugins/intel_npu/src/plugin/src/plugin.cpp b/src/plugins/intel_npu/src/plugin/src/plugin.cpp index 87db4050d31073..248a174ad9c509 100644 --- a/src/plugins/intel_npu/src/plugin/src/plugin.cpp +++ b/src/plugins/intel_npu/src/plugin/src/plugin.cpp @@ -132,30 +132,6 @@ std::map any_copy(const ov::AnyMap& params) { return result; } -size_t getFileSize(std::istream& stream) { - auto log = Logger::global().clone("getFileSize"); - if (!stream) { - OPENVINO_THROW("Stream is in bad status! Please check the passed stream status!"); - } - - const size_t streamStart = stream.tellg(); - stream.seekg(0, std::ios_base::end); - const size_t streamEnd = stream.tellg(); - stream.seekg(streamStart, std::ios_base::beg); - - log.debug("Read blob size: streamStart=%zu, streamEnd=%zu", streamStart, streamEnd); - - if (streamEnd < streamStart) { - OPENVINO_THROW("Invalid stream size: streamEnd (", - streamEnd, - ") is not larger than streamStart (", - streamStart, - ")!"); - } - - return streamEnd - streamStart; -} - void update_log_level(const std::map& propertiesMap) { auto it = propertiesMap.find(std::string(LOG_LEVEL::key())); if (it != propertiesMap.end()) { @@ -784,10 +760,22 @@ std::shared_ptr Plugin::import_model(std::istream& stream, c auto compiler = compilerAdapterFactory.getCompiler(_backends->getIEngineBackend(), localConfig); std::unique_ptr blobPtr; + std::unique_ptr storedMeta; - if (modelBuffer == nullptr) { - auto graphSize = getFileSize(stream); + if (dynamic_cast(stream.rdbuf())) { + storedMeta = read_metadata_from(stream, dynamic_cast(stream.rdbuf())->get_buffer()); + } else { + storedMeta = read_metadata_from(stream); + } + + if (storedMeta == nullptr) { + OPENVINO_THROW("Could not read metadata!"); + } else if (!storedMeta->is_compatible()) { + OPENVINO_THROW("Incompatible blob version!"); + } + auto graphSize = storedMeta->get_blob_size(); + if (modelBuffer == nullptr) { std::vector blob(graphSize); stream.read(reinterpret_cast(blob.data()), graphSize); if (!stream) { @@ -797,7 +785,7 @@ std::shared_ptr Plugin::import_model(std::istream& stream, c blobPtr = std::make_unique(std::move(blob)); } else { - blobPtr = std::make_unique(modelBuffer, stream.tellg(), 0); + blobPtr = std::make_unique(modelBuffer, storedMeta->get_ov_header_offset(), graphSize); } auto storedMeta = read_metadata_from(blob); diff --git a/src/plugins/intel_npu/tests/unit/npu/metadata_version.cpp b/src/plugins/intel_npu/tests/unit/npu/metadata_version.cpp index 6bc8e2d6436c5d..f3b637e643101b 100644 --- a/src/plugins/intel_npu/tests/unit/npu/metadata_version.cpp +++ b/src/plugins/intel_npu/tests/unit/npu/metadata_version.cpp @@ -11,38 +11,23 @@ using namespace intel_npu; using MetadataUnitTests = ::testing::Test; -struct MetadataTest : Metadata { - void set_version(uint32_t newVersion) { - _version = newVersion; - } - - void set_ov_version(const OpenvinoVersion& newVersion) { - _ovVersion = newVersion; - } -}; - TEST_F(MetadataUnitTests, readUnversionedBlob) { - std::vector blob(50, 68); + std::stringstream stream(" ELF"); - std::unique_ptr storedMeta; - ASSERT_ANY_THROW(storedMeta = read_metadata_from(blob)); + auto storedMeta = read_metadata_from(stream); ASSERT_EQ(storedMeta, nullptr); } TEST_F(MetadataUnitTests, writeAndReadMetadataFromBlob) { std::stringstream stream; size_t blobSize = 0; - auto meta = MetadataTest(); + auto meta = Metadata(stream.tellg(), blobSize); OV_ASSERT_NO_THROW(meta.write(stream)); OV_ASSERT_NO_THROW(stream.write(reinterpret_cast(&blobSize), sizeof(blobSize))); OV_ASSERT_NO_THROW(stream.write(MAGIC_BYTES.data(), MAGIC_BYTES.size())); - blobSize = stream.str().length(); - - std::vector blob(blobSize); - OV_ASSERT_NO_THROW(stream.read(reinterpret_cast(blob.data()), blobSize)); - auto storedMeta = read_metadata_from(blob); + auto storedMeta = read_metadata_from(stream); ASSERT_NE(storedMeta, nullptr); ASSERT_TRUE(storedMeta->is_compatible()); } @@ -50,7 +35,7 @@ TEST_F(MetadataUnitTests, writeAndReadMetadataFromBlob) { TEST_F(MetadataUnitTests, writeAndReadInvalidOpenvinoVersion) { size_t blobSize = 0; std::stringstream stream; - auto meta = MetadataTest(); + auto meta = Metadata(stream.tellg(), blobSize); OpenvinoVersion badOvVersion("just_some_wrong_ov_version"); meta.set_ov_version(badOvVersion); @@ -59,11 +44,7 @@ TEST_F(MetadataUnitTests, writeAndReadInvalidOpenvinoVersion) { OV_ASSERT_NO_THROW(stream.write(reinterpret_cast(&blobSize), sizeof(blobSize))); OV_ASSERT_NO_THROW(stream.write(MAGIC_BYTES.data(), MAGIC_BYTES.size())); - blobSize = stream.str().length(); - - std::vector blob(blobSize); - OV_ASSERT_NO_THROW(stream.read(reinterpret_cast(blob.data()), blobSize)); - auto storedMeta = read_metadata_from(blob); + auto storedMeta = read_metadata_from(stream); ASSERT_NE(storedMeta, nullptr); ASSERT_FALSE(storedMeta->is_compatible()); } @@ -71,7 +52,7 @@ TEST_F(MetadataUnitTests, writeAndReadInvalidOpenvinoVersion) { TEST_F(MetadataUnitTests, writeAndReadInvalidMetadataVersion) { size_t blobSize = 0; std::stringstream stream; - auto meta = Metadata(); + auto meta = Metadata(stream.tellg(), blobSize); constexpr uint32_t dummy_version = make_version(0x00007E57, 0x0000AC3D); meta.set_version(dummy_version); @@ -80,10 +61,6 @@ TEST_F(MetadataUnitTests, writeAndReadInvalidMetadataVersion) { OV_ASSERT_NO_THROW(stream.write(reinterpret_cast(&blobSize), sizeof(blobSize))); OV_ASSERT_NO_THROW(stream.write(MAGIC_BYTES.data(), MAGIC_BYTES.size())); - blobSize = stream.str().length(); - - std::vector blob(blobSize); - OV_ASSERT_NO_THROW(stream.read(reinterpret_cast(blob.data()), blobSize)); - auto storedMeta = read_metadata_from(blob); + auto storedMeta = read_metadata_from(stream); ASSERT_EQ(storedMeta, nullptr); }