Add backward compatibility to decompress bp3 files written with ADIOS…

…2.7 and older
ornladios · Jan 23, 2023 · 9712e2e · 9712e2e
1 parent e5804fe
commit 9712e2e
Show file tree

Hide file tree

Showing 2 changed files with 115 additions and 25 deletions.
diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp
@@ -125,8 +125,26 @@ void BP3Deserializer::ParseMinifooter(const BufferSTL &bufferSTL)
 
     position = bufferSize - m_MetadataSet.MiniFooterSize;
 
+    // Writer's ADIOS version
     m_Minifooter.VersionTag.assign(&buffer[position], 28);
-    position += 28;
+    position += 24;
+    m_Minifooter.ADIOSVersionMajor =
+        helper::ReadValue<uint8_t>(buffer, position,
+                                   m_Minifooter.IsLittleEndian) -
+        (uint8_t)'0';
+    m_Minifooter.ADIOSVersionMinor =
+        helper::ReadValue<uint8_t>(buffer, position,
+                                   m_Minifooter.IsLittleEndian) -
+        (uint8_t)'0';
+    m_Minifooter.ADIOSVersionPatch =
+        helper::ReadValue<uint8_t>(buffer, position,
+                                   m_Minifooter.IsLittleEndian) -
+        (uint8_t)'0';
+    m_Minifooter.ADIOSVersion = m_Minifooter.ADIOSVersionMajor * 1000000 +
+                                m_Minifooter.ADIOSVersionMinor * 1000 +
+                                m_Minifooter.ADIOSVersionPatch;
+    ++position;
+    ;
 
     m_Minifooter.PGIndexStart = helper::ReadValue<uint64_t>(
         buffer, position, m_Minifooter.IsLittleEndian);

diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc
@@ -155,8 +155,30 @@ void BP3Deserializer::SetVariableBlockInfo(
         blockOperation.PreSizeOf = sizeof(T);
 
         // read metadata from supported type and populate Info
-        std::memcpy(&blockOperation.PayloadSize, bpOpInfo.Metadata.data() + 8,
-                    8);
+        if (m_Minifooter.ADIOSVersion >= 2008000)
+        {
+            std::memcpy(&blockOperation.PayloadSize,
+                        bpOpInfo.Metadata.data() + 8, 8);
+        }
+        else
+        {
+            // Files made before 2.8 have incompatible compression operators
+            // Add backward compatible fixes here to parse compression metadata
+            // directly from the BP4 metadata format
+            std::shared_ptr<BPBackCompatOperation> bpOp =
+                SetBPBackCompatOperation(bpOpInfo.Type);
+            if (bpOp)
+            {
+                bpOp->GetMetadata(bpOpInfo.Metadata, blockOperation.Info);
+                blockOperation.PayloadSize = static_cast<size_t>(
+                    std::stoull(blockOperation.Info.at("OutputSize")));
+            }
+            else
+            {
+                std::memcpy(&blockOperation.PayloadSize,
+                            bpOpInfo.Metadata.data() + 8, 8);
+            }
+        }
 
         subStreamInfo.OperationsInfo.push_back(std::move(blockOperation));
     };
@@ -516,34 +538,43 @@ void BP3Deserializer::PostDataRead(
 {
     if (subStreamBoxInfo.OperationsInfo.size() > 0)
     {
-        const helper::BlockOperationInfo &blockOperationInfo =
-            InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo);
+        if (m_Minifooter.ADIOSVersion >= 2008000)
+        {
+            const helper::BlockOperationInfo &blockOperationInfo =
+                InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo);
 
-        const size_t preOpPayloadSize =
-            helper::GetTotalSize(blockOperationInfo.PreCount) *
-            blockOperationInfo.PreSizeOf;
-        m_ThreadBuffers[threadID][0].resize(preOpPayloadSize);
+            const size_t preOpPayloadSize =
+                helper::GetTotalSize(blockOperationInfo.PreCount) *
+                blockOperationInfo.PreSizeOf;
+            m_ThreadBuffers[threadID][0].resize(preOpPayloadSize);
 
-        // get original block back
-        char *preOpData = m_ThreadBuffers[threadID][0].data();
-        const char *postOpData = m_ThreadBuffers[threadID][1].data();
+            // get original block back
+            char *preOpData = m_ThreadBuffers[threadID][0].data();
+            const char *postOpData = m_ThreadBuffers[threadID][1].data();
 
-        std::shared_ptr<core::Operator> op = nullptr;
-        for (auto &o : blockInfo.Operations)
-        {
-            if (o->m_Category == "compress" || o->m_Category == "plugin")
+            std::shared_ptr<core::Operator> op = nullptr;
+            for (auto &o : blockInfo.Operations)
             {
-                op = o;
-                break;
+                if (o->m_Category == "compress" || o->m_Category == "plugin")
+                {
+                    op = o;
+                    break;
+                }
             }
-        }
-        core::Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData,
-                         op);
+            core::Decompress(postOpData, blockOperationInfo.PayloadSize,
+                             preOpData, op);
 
-        // clip block to match selection
-        helper::ClipVector(m_ThreadBuffers[threadID][0],
-                           subStreamBoxInfo.Seeks.first,
-                           subStreamBoxInfo.Seeks.second);
+            // clip block to match selection
+            helper::ClipVector(m_ThreadBuffers[threadID][0],
+                               subStreamBoxInfo.Seeks.first,
+                               subStreamBoxInfo.Seeks.second);
+        }
+        else
+        {
+            // Files made before 2.8 have incompatible compression operators
+            // Add backward compatible fixes in the function below
+            BackCompatDecompress(subStreamBoxInfo, threadID);
+        }
     }
 
 #ifdef ADIOS2_HAVE_ENDIAN_REVERSE
@@ -609,6 +640,47 @@ void BP3Deserializer::PostDataRead(
             endianReverse);
     }
 }
+void BP3Deserializer::BackCompatDecompress(
+    const helper::SubStreamBoxInfo &subStreamBoxInfo, const size_t threadID)
+{
+    // Files made before 2.8 have incompatible compression operators
+    // Add backward compatible fixes here
+    const helper::BlockOperationInfo &blockOperationInfo =
+        InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo);
+
+    const size_t preOpPayloadSize =
+        helper::GetTotalSize(blockOperationInfo.PreCount) *
+        blockOperationInfo.PreSizeOf;
+    m_ThreadBuffers[threadID][0].resize(preOpPayloadSize);
+
+    std::string opType = blockOperationInfo.Info.at("Type");
+
+    // get original block back
+    char *preOpData = m_ThreadBuffers[threadID][0].data();
+    const char *postOpData = m_ThreadBuffers[threadID][1].data();
+    // get the right bp3Op
+    std::shared_ptr<BPBackCompatOperation> bp3Op =
+        SetBPBackCompatOperation(opType);
+
+    if (bp3Op)
+    {
+        bp3Op->GetData(postOpData, blockOperationInfo, preOpData);
+        // clip block to match selection
+        helper::ClipVector(m_ThreadBuffers[threadID][0],
+                           subStreamBoxInfo.Seeks.first,
+                           subStreamBoxInfo.Seeks.second);
+    }
+    else
+    {
+        helper::Throw<std::runtime_error>(
+            "Toolkit", "format::bp::BP3Deserializer", "PostDataRead",
+            "This file was created by pre-ADIOS 2.8.0 using "
+            "compression type " +
+                opType +
+                ", for which there is no backward compatible reader in this "
+                "ADIOS version");
+    }
+}
 
 template <class T>
 std::map<size_t, std::vector<typename core::Variable<T>::BPInfo>>