From 9712e2e698ab0dea264657d7ad4e72289439a620 Mon Sep 17 00:00:00 2001 From: lizdulac Date: Mon, 23 Jan 2023 16:27:49 -0500 Subject: [PATCH] Add backward compatibility to decompress bp3 files written with ADIOS2.7 and older --- .../toolkit/format/bp/bp3/BP3Deserializer.cpp | 20 ++- .../toolkit/format/bp/bp3/BP3Deserializer.tcc | 120 ++++++++++++++---- 2 files changed, 115 insertions(+), 25 deletions(-) diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp index 41d99d46ff..d0603ae2bb 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp @@ -125,8 +125,26 @@ void BP3Deserializer::ParseMinifooter(const BufferSTL &bufferSTL) position = bufferSize - m_MetadataSet.MiniFooterSize; + // Writer's ADIOS version m_Minifooter.VersionTag.assign(&buffer[position], 28); - position += 28; + position += 24; + m_Minifooter.ADIOSVersionMajor = + helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian) - + (uint8_t)'0'; + m_Minifooter.ADIOSVersionMinor = + helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian) - + (uint8_t)'0'; + m_Minifooter.ADIOSVersionPatch = + helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian) - + (uint8_t)'0'; + m_Minifooter.ADIOSVersion = m_Minifooter.ADIOSVersionMajor * 1000000 + + m_Minifooter.ADIOSVersionMinor * 1000 + + m_Minifooter.ADIOSVersionPatch; + ++position; + ; m_Minifooter.PGIndexStart = helper::ReadValue( buffer, position, m_Minifooter.IsLittleEndian); diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc index 32b66e3592..3ba7ffbf2d 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc @@ -155,8 +155,30 @@ void BP3Deserializer::SetVariableBlockInfo( blockOperation.PreSizeOf = sizeof(T); // read metadata from supported type and populate Info - std::memcpy(&blockOperation.PayloadSize, bpOpInfo.Metadata.data() + 8, - 8); + if (m_Minifooter.ADIOSVersion >= 2008000) + { + std::memcpy(&blockOperation.PayloadSize, + bpOpInfo.Metadata.data() + 8, 8); + } + else + { + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes here to parse compression metadata + // directly from the BP4 metadata format + std::shared_ptr bpOp = + SetBPBackCompatOperation(bpOpInfo.Type); + if (bpOp) + { + bpOp->GetMetadata(bpOpInfo.Metadata, blockOperation.Info); + blockOperation.PayloadSize = static_cast( + std::stoull(blockOperation.Info.at("OutputSize"))); + } + else + { + std::memcpy(&blockOperation.PayloadSize, + bpOpInfo.Metadata.data() + 8, 8); + } + } subStreamInfo.OperationsInfo.push_back(std::move(blockOperation)); }; @@ -516,34 +538,43 @@ void BP3Deserializer::PostDataRead( { if (subStreamBoxInfo.OperationsInfo.size() > 0) { - const helper::BlockOperationInfo &blockOperationInfo = - InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); + if (m_Minifooter.ADIOSVersion >= 2008000) + { + const helper::BlockOperationInfo &blockOperationInfo = + InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); - const size_t preOpPayloadSize = - helper::GetTotalSize(blockOperationInfo.PreCount) * - blockOperationInfo.PreSizeOf; - m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); + const size_t preOpPayloadSize = + helper::GetTotalSize(blockOperationInfo.PreCount) * + blockOperationInfo.PreSizeOf; + m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); - // get original block back - char *preOpData = m_ThreadBuffers[threadID][0].data(); - const char *postOpData = m_ThreadBuffers[threadID][1].data(); + // get original block back + char *preOpData = m_ThreadBuffers[threadID][0].data(); + const char *postOpData = m_ThreadBuffers[threadID][1].data(); - std::shared_ptr op = nullptr; - for (auto &o : blockInfo.Operations) - { - if (o->m_Category == "compress" || o->m_Category == "plugin") + std::shared_ptr op = nullptr; + for (auto &o : blockInfo.Operations) { - op = o; - break; + if (o->m_Category == "compress" || o->m_Category == "plugin") + { + op = o; + break; + } } - } - core::Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData, - op); + core::Decompress(postOpData, blockOperationInfo.PayloadSize, + preOpData, op); - // clip block to match selection - helper::ClipVector(m_ThreadBuffers[threadID][0], - subStreamBoxInfo.Seeks.first, - subStreamBoxInfo.Seeks.second); + // clip block to match selection + helper::ClipVector(m_ThreadBuffers[threadID][0], + subStreamBoxInfo.Seeks.first, + subStreamBoxInfo.Seeks.second); + } + else + { + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes in the function below + BackCompatDecompress(subStreamBoxInfo, threadID); + } } #ifdef ADIOS2_HAVE_ENDIAN_REVERSE @@ -609,6 +640,47 @@ void BP3Deserializer::PostDataRead( endianReverse); } } +void BP3Deserializer::BackCompatDecompress( + const helper::SubStreamBoxInfo &subStreamBoxInfo, const size_t threadID) +{ + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes here + const helper::BlockOperationInfo &blockOperationInfo = + InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); + + const size_t preOpPayloadSize = + helper::GetTotalSize(blockOperationInfo.PreCount) * + blockOperationInfo.PreSizeOf; + m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); + + std::string opType = blockOperationInfo.Info.at("Type"); + + // get original block back + char *preOpData = m_ThreadBuffers[threadID][0].data(); + const char *postOpData = m_ThreadBuffers[threadID][1].data(); + // get the right bp3Op + std::shared_ptr bp3Op = + SetBPBackCompatOperation(opType); + + if (bp3Op) + { + bp3Op->GetData(postOpData, blockOperationInfo, preOpData); + // clip block to match selection + helper::ClipVector(m_ThreadBuffers[threadID][0], + subStreamBoxInfo.Seeks.first, + subStreamBoxInfo.Seeks.second); + } + else + { + helper::Throw( + "Toolkit", "format::bp::BP3Deserializer", "PostDataRead", + "This file was created by pre-ADIOS 2.8.0 using " + "compression type " + + opType + + ", for which there is no backward compatible reader in this " + "ADIOS version"); + } +} template std::map::BPInfo>>