Revert "Migrate undos to dedicated DB (#1381)"

This reverts commit 67e12c8.

src/flushablestorage.h

@@ -503,9 +503,6 @@ class CStorageView {
             }
         }
     }
-    CFlushableStorageKV& GetStorage() {
-        return static_cast<CFlushableStorageKV&>(DB());
-    }
 
     virtual bool Flush() { return DB().Flush(); }
     void Discard() { DB().Discard(); }

src/init.cpp

@@ -1340,48 +1340,6 @@ void SetupAnchorSPVDatabases(bool resync) {
     }
 }
 
-
-void MigrateDBs()
-{
-    auto it = pundosView->LowerBound<CUndosView::ByMultiUndoKey>(UndoSourceKey{});
-    if (it.Valid()) {
-        return;
-    }
-
-    auto time = GetTimeMillis();
-
-    // Migrate Undos
-    auto migrationStarted{false};
-    auto mnview(*pcustomcsview);
-    pcustomcsview->ForEachUndo([&](const UndoKey& key, const CUndo& undo){
-        if (!migrationStarted) {
-            migrationStarted = true;
-            LogPrintf("Migrating undo entries, might take a while...\n");
-        }
-
-        pundosView->SetUndo({{key.height, key.txid}, UndoSource::CustomView}, undo);
-        mnview.DelUndo(key);
-
-        return true;
-    });
-
-    if (migrationStarted) {
-        pundosView->Flush();
-        pundosDB->Flush();
-
-        auto& map = mnview.GetStorage().GetRaw();
-        const auto compactBegin = map.begin()->first;
-        const auto compactEnd = map.rbegin()->first;
-        mnview.Flush();
-        pcustomcsview->Flush();
-        pcustomcsDB->Flush();
-        pcustomcsDB->Compact(compactBegin, compactEnd);
-
-        LogPrintf("Migrating undo data finished.\n");
-        LogPrint(BCLog::BENCH, "    - Migrating undo data takes: %dms\n", GetTimeMillis() - time);
-    }
-}
-
 bool SetupInterruptArg(const std::string &argName, std::string &hashStore, int &heightStore) {
     // Experimental: Block height or hash to invalidate on and stop sync
     auto val = gArgs.GetArg(argName, "");
@@ -1759,7 +1717,7 @@ bool AppInitMain(InitInterfaces& interfaces)
                 pcustomcsDB.reset();
                 pcustomcsDB = std::make_unique<CStorageLevelDB>(GetDataDir() / "enhancedcs", nCustomCacheSize, false, fReset || fReindexChainState);
                 pcustomcsview.reset();
-                pcustomcsview = std::make_unique<CCustomCSView>(*pcustomcsDB);
+                pcustomcsview = std::make_unique<CCustomCSView>(*pcustomcsDB.get());
 
                 if (!fReset && !fReindexChainState) {
                     if (!pcustomcsDB->IsEmpty() && pcustomcsview->GetDbVersion() != CCustomCSView::DbVersion) {
@@ -1791,15 +1749,6 @@ bool AppInitMain(InitInterfaces& interfaces)
                     pvaultHistoryDB = std::make_unique<CVaultHistoryStorage>(GetDataDir() / "vault", nCustomCacheSize, false, fReset || fReindexChainState);
                 }
 
-                // Create Undo DB
-                pundosDB.reset();
-                pundosDB = std::make_unique<CStorageLevelDB>(GetDataDir() / "undos", nCustomCacheSize, false, fReset || fReindexChainState);
-                pundosView.reset();
-                pundosView = std::make_unique<CUndosView>(*pundosDB);
-
-                // Migrate FutureSwaps and Undos to their own new DBs.
-                MigrateDBs();
-
                 // If necessary, upgrade from older database format.
                 // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate
                 if (!::ChainstateActive().CoinsDB().Upgrade()) {
@@ -1808,7 +1757,7 @@ bool AppInitMain(InitInterfaces& interfaces)
                 }
 
                 // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate
-                if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get(), pundosView.get())) {
+                if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get())) {
                     strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated;
                     break;
                 }

src/masternodes/masternodes.cpp

@@ -709,10 +709,7 @@ void CSettingsView::SetDexStatsLastHeight(const int32_t height)
 
 std::optional<int32_t> CSettingsView::GetDexStatsLastHeight()
 {
-    if (auto res = ReadBy<KVSettings, int32_t>(DEX_STATS_LAST_HEIGHT)) {
-        return res;
-    }
-    return {};
+    return ReadBy<KVSettings, int32_t>(DEX_STATS_LAST_HEIGHT);
 }
 
 void CSettingsView::SetDexStatsEnabled(const bool enabled)
@@ -881,6 +878,16 @@ void CCustomCSView::CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::Ancho
     }
 }
 
+void CCustomCSView::OnUndoTx(uint256 const & txid, uint32_t height)
+{
+    const auto undo = GetUndo(UndoKey{height, txid});
+    if (!undo) {
+        return; // not custom tx, or no changes done
+    }
+    CUndo::Revert(GetStorage(), *undo); // revert the changes of this tx
+    DelUndo(UndoKey{height, txid}); // erase undo data, it served its purpose
+}
+
 bool CCustomCSView::CanSpend(const uint256 & txId, int height) const
 {
     auto node = GetMasternode(txId);
@@ -1078,31 +1085,28 @@ Res CCustomCSView::PopulateCollateralData(CCollateralLoans& result, CVaultId con
     return Res::Ok();
 }
 
-uint256 CCustomCSView::MerkleRoot(CUndosView& undo) {
+uint256 CCustomCSView::MerkleRoot() {
     auto rawMap = GetStorage().GetRaw();
     if (rawMap.empty()) {
         return {};
     }
     auto isAttributes = [](const TBytes& key) {
+        MapKV map = {std::make_pair(key, TBytes{})};
         // Attributes should not be part of merkle root
         static const std::string attributes("ATTRIBUTES");
-        MapKV map = {std::make_pair(key, TBytes{})};
         auto it = NewKVIterator<CGovView::ByName>(attributes, map);
         return it.Valid() && it.Key() == attributes;
     };
-    auto& undoStorage = undo.GetStorage();
-    auto& undoMap = undoStorage.GetRaw();
-    auto it = NewKVIterator<CUndosView::ByMultiUndoKey>(UndoSourceKey{}, undoMap);
+
+    auto it = NewKVIterator<CUndosView::ByUndoKey>(UndoKey{}, rawMap);
     for (; it.Valid(); it.Next()) {
-        if (it.Key().key == UndoSource::CustomView) {
-            CUndo value = it.Value();
-            auto& map = value.before;
-            for (auto it = map.begin(); it != map.end();) {
-                isAttributes(it->first) ? map.erase(it++) : ++it;
-            }
-            auto key = std::make_pair(CUndosBaseView::ByUndoKey::prefix(), static_cast<const UndoKey&>(it.Key()));
-            rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value);
+        CUndo value = it.Value();
+        auto& map = value.before;
+        for (auto it = map.begin(); it != map.end();) {
+            isAttributes(it->first) ? map.erase(it++) : ++it;
         }
+        auto key = std::make_pair(CUndosView::ByUndoKey::prefix(), static_cast<const UndoKey&>(it.Key()));
+        rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value);
     }
 
     std::vector<uint256> hashes;

src/masternodes/masternodes.h

@@ -390,7 +390,7 @@ class CCustomCSView
         , public CTokensView
         , public CAccountsView
         , public CCommunityBalancesView
-        , public CUndosBaseView
+        , public CUndosView
         , public CPoolPairView
         , public CGovView
         , public CAnchorConfirmsView
@@ -411,7 +411,7 @@ class CCustomCSView
             CTokensView             ::  ID, Symbol, CreationTx, LastDctId,
             CAccountsView           ::  ByBalanceKey, ByHeightKey, ByFuturesSwapKey,
             CCommunityBalancesView  ::  ById,
-            CUndosBaseView          ::  ByUndoKey,
+            CUndosView              ::  ByUndoKey,
             CPoolPairView           ::  ByID, ByPair, ByShare, ByIDPair, ByPoolSwap, ByReserves, ByRewardPct, ByRewardLoanPct,
                                         ByPoolReward, ByDailyReward, ByCustomReward, ByTotalLiquidity, ByDailyLoanReward,
                                         ByPoolLoanReward, ByTokenDexFeePct,
@@ -461,6 +461,9 @@ class CCustomCSView
     /// @todo newbase move to networking?
     void CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::AnchorRec* anchor, const uint256 & btcTxHash, const CKey &masternodeKey);
 
+    // simplified version of undo, without any unnecessary undo data
+    void OnUndoTx(uint256 const & txid, uint32_t height);
+
     bool CanSpend(const uint256 & txId, int height) const;
 
     bool CalculateOwnerRewards(CScript const & owner, uint32_t height);
@@ -484,7 +487,12 @@ class CCustomCSView
     void SetGlobalCustomTxExpiration(const uint32_t height);
     uint32_t GetGlobalCustomTxExpiration() const;
 
-    uint256 MerkleRoot(CUndosView& undo);
+    uint256 MerkleRoot();
+
+    // we construct it as it
+    CFlushableStorageKV& GetStorage() {
+        return static_cast<CFlushableStorageKV&>(DB());
+    }
 
     virtual CAccountHistoryStorage* GetAccountHistoryStore();
     CVaultHistoryStorage* GetVaultHistoryStore();

src/masternodes/mn_checks.cpp

@@ -3774,7 +3774,15 @@ Res ApplyCustomTx(CCustomCSView& mnview, const CCoinsViewCache& coins, const CTr
         return res;
     }
 
+    // construct undo
+    auto& flushable = view.GetStorage();
+    auto undo = CUndo::Construct(mnview.GetStorage(), flushable.GetRaw());
+    // flush changes
     view.Flush();
+    // write undo
+    if (!undo.before.empty()) {
+        mnview.SetUndo(UndoKey{height, tx.GetHash()}, undo);
+    }
     return res;
 }
 

src/masternodes/undo.h

@@ -12,11 +12,6 @@
 #include <serialize_optional.h>
 #include <flushablestorage.h>
 
-// Enum for future support of multiple sources of undo data, not just CustomView.
-enum UndoSource : uint8_t {
-    CustomView = 0,
-};
-
 struct UndoKey {
     uint32_t height; // height is there to be able to prune older undos using lexicographic iteration
     uint256 txid;
@@ -30,18 +25,6 @@ struct UndoKey {
     }
 };
 
-struct UndoSourceKey : UndoKey {
-    uint8_t key;
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITEAS(UndoKey, *this);
-        READWRITE(key);
-    }
-};
-
 struct CUndo {
     MapKV before;
 

src/masternodes/undos.cpp

@@ -4,61 +4,29 @@
 
 #include <masternodes/undos.h>
 
-void CUndosBaseView::ForEachUndo(std::function<bool(UndoKey const &, CLazySerialize<CUndo>)> callback, UndoKey const & start)
+void CUndosView::ForEachUndo(std::function<bool(UndoKey const &, CLazySerialize<CUndo>)> callback, UndoKey const & start)
 {
     ForEach<ByUndoKey, UndoKey, CUndo>(callback, start);
 }
 
-Res CUndosBaseView::DelUndo(UndoKey const & key)
+Res CUndosView::SetUndo(UndoKey const & key, CUndo const & undo)
 {
-    EraseBy<ByUndoKey>(key);
+    WriteBy<ByUndoKey>(key, undo);
     return Res::Ok();
 }
 
-void CUndosView::ForEachUndo(std::function<bool(const UndoSourceKey &, CLazySerialize<CUndo>)> callback, const UndoSourceKey& start)
+Res CUndosView::DelUndo(UndoKey const & key)
 {
-    ForEach<ByMultiUndoKey, UndoSourceKey, CUndo>(callback, start);
-}
-
-void CUndosView::AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height)
-{
-    auto& flushable = cache.GetStorage();
-    auto& rawMap = flushable.GetRaw();
-    if (!rawMap.empty()) {
-        SetUndo({{height, txid}, key}, CUndo::Construct(source.GetStorage(), rawMap));
-    }
-}
-
-Res CUndosView::SetUndo(UndoSourceKey const & key, CUndo const & undo)
-{
-    WriteBy<ByMultiUndoKey>(key, undo);
+    EraseBy<ByUndoKey>(key);
     return Res::Ok();
 }
 
-void CUndosView::OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height)
-{
-    const auto undo = GetUndo({{height, txid}, key});
-    if (!undo) {
-        return; // not custom tx, or no changes done
-    }
-    CUndo::Revert(source.GetStorage(), *undo); // revert the changes of this tx
-    DelUndo({{height, txid}, key}); // erase undo data, it served its purpose
-}
-
-std::optional<CUndo> CUndosView::GetUndo(UndoSourceKey const & key) const
+std::optional<CUndo> CUndosView::GetUndo(UndoKey const & key) const
 {
     CUndo val;
-    if (ReadBy<ByMultiUndoKey>(key, val)) {
+    bool ok = ReadBy<ByUndoKey>(key, val);
+    if (ok) {
         return val;
     }
     return {};
 }
-
-Res CUndosView::DelUndo(const UndoSourceKey & key)
-{
-    EraseBy<ByMultiUndoKey>(key);
-    return Res::Ok();
-}
-
-std::unique_ptr<CStorageLevelDB> pundosDB;
-std::unique_ptr<CUndosView> pundosView;

src/masternodes/undos.h

@@ -9,36 +9,17 @@
 #include <flushablestorage.h>
 #include <masternodes/res.h>
 
-class CUndosBaseView : public virtual CStorageView {
+class CUndosView : public virtual CStorageView {
 public:
     void ForEachUndo(std::function<bool(UndoKey const &, CLazySerialize<CUndo>)> callback, UndoKey const & start = {});
 
-    Res DelUndo(const UndoKey & key);
+    std::optional<CUndo> GetUndo(UndoKey const & key) const;
+    Res SetUndo(UndoKey const & key, CUndo const & undo);
+    Res DelUndo(UndoKey const & key);
 
     // tags
     struct ByUndoKey { static constexpr uint8_t prefix() { return 'u'; } };
 };
 
-class CUndosView : public CStorageView
-{
-public:
-    CUndosView(CUndosView& other) : CStorageView(new CFlushableStorageKV(other.DB())) {}
-    explicit CUndosView(CStorageKV& st) : CStorageView(new CFlushableStorageKV(st)) {}
-
-    void ForEachUndo(std::function<bool(const UndoSourceKey &, CLazySerialize<CUndo>)> callback, const UndoSourceKey& start = {});
-
-    [[nodiscard]] std::optional<CUndo> GetUndo(UndoSourceKey const & key) const;
-    Res SetUndo(const UndoSourceKey& key, const CUndo& undo);
-    Res DelUndo(const UndoSourceKey & key);
-
-    void AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height);
-    void OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height);
-
-    // tags
-    struct ByMultiUndoKey { static constexpr uint8_t prefix() { return 'n'; } };
-};
-
-extern std::unique_ptr<CStorageLevelDB> pundosDB;
-extern std::unique_ptr<CUndosView> pundosView;
 
 #endif //DEFI_MASTERNODES_UNDOS_H

src/miner.cpp

@@ -213,7 +213,6 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc
     int nPackagesSelected = 0;
     int nDescendantsUpdated = 0;
     CCustomCSView mnview(*pcustomcsview);
-    CUndosView undosView(*pundosView);
     if (!blockTime) {
         UpdateTime(pblock, consensus, pindexPrev); // update time before tx packaging
     }
@@ -355,7 +354,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc
     && nHeight < chainparams.GetConsensus().EunosKampungHeight) {
         // includes coinbase account changes
         ApplyGeneralCoinbaseTx(mnview, *(pblock->vtx[0]), nHeight, nFees, chainparams.GetConsensus());
-        pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot(undosView));
+        pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot());
     }
 
     LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", 0.001 * (nTime1 - nTimeStart), nPackagesSelected, nDescendantsUpdated, 0.001 * (nTime2 - nTime1), 0.001 * (nTime2 - nTimeStart));

src/test/setup_common.cpp

@@ -113,11 +113,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha
 
         pcustomcsDB.reset();
         pcustomcsDB = std::make_unique<CStorageLevelDB>(GetDataDir() / "enhancedcs", nMinDbCache << 20, true, true);
-        pcustomcsview = std::make_unique<CCustomCSView>(*pcustomcsDB);
-
-        pundosDB.reset();
-        pundosDB = std::make_unique<CStorageLevelDB>(GetDataDir() / "undos", nMinDbCache << 20, true, true);
-        pundosView = std::make_unique<CUndosView>(*pundosDB);
+        pcustomcsview = std::make_unique<CCustomCSView>(*pcustomcsDB.get());
 
         panchorauths.reset();
         panchorauths = std::make_unique<CAnchorAuthIndex>();