From 67e12c85d401aef58b846bdc4dd318166213099d Mon Sep 17 00:00:00 2001 From: Peter John Bushnell Date: Tue, 12 Jul 2022 13:03:22 +0100 Subject: [PATCH 1/2] Migrate undos to dedicated DB (#1381) --- src/flushablestorage.h | 3 + src/init.cpp | 55 ++++++- src/masternodes/masternodes.cpp | 38 ++--- src/masternodes/masternodes.h | 14 +- src/masternodes/mn_checks.cpp | 8 - src/masternodes/undo.h | 17 ++ src/masternodes/undos.cpp | 48 +++++- src/masternodes/undos.h | 27 ++- src/miner.cpp | 3 +- src/test/setup_common.cpp | 6 +- src/test/storage_tests.cpp | 29 ++-- src/validation.cpp | 155 ++++++++++-------- src/validation.h | 12 +- .../test_framework/test_framework.py | 1 + 14 files changed, 277 insertions(+), 139 deletions(-) diff --git a/src/flushablestorage.h b/src/flushablestorage.h index fd41d96a0d9..3d50ea0c85b 100644 --- a/src/flushablestorage.h +++ b/src/flushablestorage.h @@ -503,6 +503,9 @@ class CStorageView { } } } + CFlushableStorageKV& GetStorage() { + return static_cast(DB()); + } virtual bool Flush() { return DB().Flush(); } void Discard() { DB().Discard(); } diff --git a/src/init.cpp b/src/init.cpp index ca0518ee6e4..2d294623622 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1340,6 +1340,48 @@ void SetupAnchorSPVDatabases(bool resync) { } } + +void MigrateDBs() +{ + auto it = pundosView->LowerBound(UndoSourceKey{}); + if (it.Valid()) { + return; + } + + auto time = GetTimeMillis(); + + // Migrate Undos + auto migrationStarted{false}; + auto mnview(*pcustomcsview); + pcustomcsview->ForEachUndo([&](const UndoKey& key, const CUndo& undo){ + if (!migrationStarted) { + migrationStarted = true; + LogPrintf("Migrating undo entries, might take a while...\n"); + } + + pundosView->SetUndo({{key.height, key.txid}, UndoSource::CustomView}, undo); + mnview.DelUndo(key); + + return true; + }); + + if (migrationStarted) { + pundosView->Flush(); + pundosDB->Flush(); + + auto& map = mnview.GetStorage().GetRaw(); + const auto compactBegin = map.begin()->first; + const auto compactEnd = map.rbegin()->first; + mnview.Flush(); + pcustomcsview->Flush(); + pcustomcsDB->Flush(); + pcustomcsDB->Compact(compactBegin, compactEnd); + + LogPrintf("Migrating undo data finished.\n"); + LogPrint(BCLog::BENCH, " - Migrating undo data takes: %dms\n", GetTimeMillis() - time); + } +} + bool SetupInterruptArg(const std::string &argName, std::string &hashStore, int &heightStore) { // Experimental: Block height or hash to invalidate on and stop sync auto val = gArgs.GetArg(argName, ""); @@ -1717,7 +1759,7 @@ bool AppInitMain(InitInterfaces& interfaces) pcustomcsDB.reset(); pcustomcsDB = std::make_unique(GetDataDir() / "enhancedcs", nCustomCacheSize, false, fReset || fReindexChainState); pcustomcsview.reset(); - pcustomcsview = std::make_unique(*pcustomcsDB.get()); + pcustomcsview = std::make_unique(*pcustomcsDB); if (!fReset && !fReindexChainState) { if (!pcustomcsDB->IsEmpty() && pcustomcsview->GetDbVersion() != CCustomCSView::DbVersion) { @@ -1749,6 +1791,15 @@ bool AppInitMain(InitInterfaces& interfaces) pvaultHistoryDB = std::make_unique(GetDataDir() / "vault", nCustomCacheSize, false, fReset || fReindexChainState); } + // Create Undo DB + pundosDB.reset(); + pundosDB = std::make_unique(GetDataDir() / "undos", nCustomCacheSize, false, fReset || fReindexChainState); + pundosView.reset(); + pundosView = std::make_unique(*pundosDB); + + // Migrate FutureSwaps and Undos to their own new DBs. + MigrateDBs(); + // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!::ChainstateActive().CoinsDB().Upgrade()) { @@ -1757,7 +1808,7 @@ bool AppInitMain(InitInterfaces& interfaces) } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get())) { + if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get(), pundosView.get())) { strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; break; } diff --git a/src/masternodes/masternodes.cpp b/src/masternodes/masternodes.cpp index 1ca14e3a589..915a2df6231 100644 --- a/src/masternodes/masternodes.cpp +++ b/src/masternodes/masternodes.cpp @@ -709,7 +709,10 @@ void CSettingsView::SetDexStatsLastHeight(const int32_t height) std::optional CSettingsView::GetDexStatsLastHeight() { - return ReadBy(DEX_STATS_LAST_HEIGHT); + if (auto res = ReadBy(DEX_STATS_LAST_HEIGHT)) { + return res; + } + return {}; } void CSettingsView::SetDexStatsEnabled(const bool enabled) @@ -878,16 +881,6 @@ void CCustomCSView::CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::Ancho } } -void CCustomCSView::OnUndoTx(uint256 const & txid, uint32_t height) -{ - const auto undo = GetUndo(UndoKey{height, txid}); - if (!undo) { - return; // not custom tx, or no changes done - } - CUndo::Revert(GetStorage(), *undo); // revert the changes of this tx - DelUndo(UndoKey{height, txid}); // erase undo data, it served its purpose -} - bool CCustomCSView::CanSpend(const uint256 & txId, int height) const { auto node = GetMasternode(txId); @@ -1085,28 +1078,31 @@ Res CCustomCSView::PopulateCollateralData(CCollateralLoans& result, CVaultId con return Res::Ok(); } -uint256 CCustomCSView::MerkleRoot() { +uint256 CCustomCSView::MerkleRoot(CUndosView& undo) { auto rawMap = GetStorage().GetRaw(); if (rawMap.empty()) { return {}; } auto isAttributes = [](const TBytes& key) { - MapKV map = {std::make_pair(key, TBytes{})}; // Attributes should not be part of merkle root static const std::string attributes("ATTRIBUTES"); + MapKV map = {std::make_pair(key, TBytes{})}; auto it = NewKVIterator(attributes, map); return it.Valid() && it.Key() == attributes; }; - - auto it = NewKVIterator(UndoKey{}, rawMap); + auto& undoStorage = undo.GetStorage(); + auto& undoMap = undoStorage.GetRaw(); + auto it = NewKVIterator(UndoSourceKey{}, undoMap); for (; it.Valid(); it.Next()) { - CUndo value = it.Value(); - auto& map = value.before; - for (auto it = map.begin(); it != map.end();) { - isAttributes(it->first) ? map.erase(it++) : ++it; + if (it.Key().key == UndoSource::CustomView) { + CUndo value = it.Value(); + auto& map = value.before; + for (auto it = map.begin(); it != map.end();) { + isAttributes(it->first) ? map.erase(it++) : ++it; + } + auto key = std::make_pair(CUndosBaseView::ByUndoKey::prefix(), static_cast(it.Key())); + rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value); } - auto key = std::make_pair(CUndosView::ByUndoKey::prefix(), static_cast(it.Key())); - rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value); } std::vector hashes; diff --git a/src/masternodes/masternodes.h b/src/masternodes/masternodes.h index a9db52b6e02..6f6ea335e20 100644 --- a/src/masternodes/masternodes.h +++ b/src/masternodes/masternodes.h @@ -390,7 +390,7 @@ class CCustomCSView , public CTokensView , public CAccountsView , public CCommunityBalancesView - , public CUndosView + , public CUndosBaseView , public CPoolPairView , public CGovView , public CAnchorConfirmsView @@ -411,7 +411,7 @@ class CCustomCSView CTokensView :: ID, Symbol, CreationTx, LastDctId, CAccountsView :: ByBalanceKey, ByHeightKey, ByFuturesSwapKey, CCommunityBalancesView :: ById, - CUndosView :: ByUndoKey, + CUndosBaseView :: ByUndoKey, CPoolPairView :: ByID, ByPair, ByShare, ByIDPair, ByPoolSwap, ByReserves, ByRewardPct, ByRewardLoanPct, ByPoolReward, ByDailyReward, ByCustomReward, ByTotalLiquidity, ByDailyLoanReward, ByPoolLoanReward, ByTokenDexFeePct, @@ -461,9 +461,6 @@ class CCustomCSView /// @todo newbase move to networking? void CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::AnchorRec* anchor, const uint256 & btcTxHash, const CKey &masternodeKey); - // simplified version of undo, without any unnecessary undo data - void OnUndoTx(uint256 const & txid, uint32_t height); - bool CanSpend(const uint256 & txId, int height) const; bool CalculateOwnerRewards(CScript const & owner, uint32_t height); @@ -487,12 +484,7 @@ class CCustomCSView void SetGlobalCustomTxExpiration(const uint32_t height); uint32_t GetGlobalCustomTxExpiration() const; - uint256 MerkleRoot(); - - // we construct it as it - CFlushableStorageKV& GetStorage() { - return static_cast(DB()); - } + uint256 MerkleRoot(CUndosView& undo); virtual CAccountHistoryStorage* GetAccountHistoryStore(); CVaultHistoryStorage* GetVaultHistoryStore(); diff --git a/src/masternodes/mn_checks.cpp b/src/masternodes/mn_checks.cpp index 9589dd6a9d6..d0b447db2dd 100644 --- a/src/masternodes/mn_checks.cpp +++ b/src/masternodes/mn_checks.cpp @@ -3774,15 +3774,7 @@ Res ApplyCustomTx(CCustomCSView& mnview, const CCoinsViewCache& coins, const CTr return res; } - // construct undo - auto& flushable = view.GetStorage(); - auto undo = CUndo::Construct(mnview.GetStorage(), flushable.GetRaw()); - // flush changes view.Flush(); - // write undo - if (!undo.before.empty()) { - mnview.SetUndo(UndoKey{height, tx.GetHash()}, undo); - } return res; } diff --git a/src/masternodes/undo.h b/src/masternodes/undo.h index b9f6e1e967c..27c5d596769 100644 --- a/src/masternodes/undo.h +++ b/src/masternodes/undo.h @@ -12,6 +12,11 @@ #include #include +// Enum for future support of multiple sources of undo data, not just CustomView. +enum UndoSource : uint8_t { + CustomView = 0, +}; + struct UndoKey { uint32_t height; // height is there to be able to prune older undos using lexicographic iteration uint256 txid; @@ -25,6 +30,18 @@ struct UndoKey { } }; +struct UndoSourceKey : UndoKey { + uint8_t key; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITEAS(UndoKey, *this); + READWRITE(key); + } +}; + struct CUndo { MapKV before; diff --git a/src/masternodes/undos.cpp b/src/masternodes/undos.cpp index 47bd1d9bfdb..40a12efe81c 100644 --- a/src/masternodes/undos.cpp +++ b/src/masternodes/undos.cpp @@ -4,29 +4,61 @@ #include -void CUndosView::ForEachUndo(std::function)> callback, UndoKey const & start) +void CUndosBaseView::ForEachUndo(std::function)> callback, UndoKey const & start) { ForEach(callback, start); } -Res CUndosView::SetUndo(UndoKey const & key, CUndo const & undo) +Res CUndosBaseView::DelUndo(UndoKey const & key) { - WriteBy(key, undo); + EraseBy(key); return Res::Ok(); } -Res CUndosView::DelUndo(UndoKey const & key) +void CUndosView::ForEachUndo(std::function)> callback, const UndoSourceKey& start) { - EraseBy(key); + ForEach(callback, start); +} + +void CUndosView::AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height) +{ + auto& flushable = cache.GetStorage(); + auto& rawMap = flushable.GetRaw(); + if (!rawMap.empty()) { + SetUndo({{height, txid}, key}, CUndo::Construct(source.GetStorage(), rawMap)); + } +} + +Res CUndosView::SetUndo(UndoSourceKey const & key, CUndo const & undo) +{ + WriteBy(key, undo); return Res::Ok(); } -std::optional CUndosView::GetUndo(UndoKey const & key) const +void CUndosView::OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height) +{ + const auto undo = GetUndo({{height, txid}, key}); + if (!undo) { + return; // not custom tx, or no changes done + } + CUndo::Revert(source.GetStorage(), *undo); // revert the changes of this tx + DelUndo({{height, txid}, key}); // erase undo data, it served its purpose +} + +std::optional CUndosView::GetUndo(UndoSourceKey const & key) const { CUndo val; - bool ok = ReadBy(key, val); - if (ok) { + if (ReadBy(key, val)) { return val; } return {}; } + +Res CUndosView::DelUndo(const UndoSourceKey & key) +{ + EraseBy(key); + return Res::Ok(); +} + +std::unique_ptr pundosDB; +std::unique_ptr pundosView; diff --git a/src/masternodes/undos.h b/src/masternodes/undos.h index bf9ebe1f1cd..dbb3b19ba45 100644 --- a/src/masternodes/undos.h +++ b/src/masternodes/undos.h @@ -9,17 +9,36 @@ #include #include -class CUndosView : public virtual CStorageView { +class CUndosBaseView : public virtual CStorageView { public: void ForEachUndo(std::function)> callback, UndoKey const & start = {}); - std::optional GetUndo(UndoKey const & key) const; - Res SetUndo(UndoKey const & key, CUndo const & undo); - Res DelUndo(UndoKey const & key); + Res DelUndo(const UndoKey & key); // tags struct ByUndoKey { static constexpr uint8_t prefix() { return 'u'; } }; }; +class CUndosView : public CStorageView +{ +public: + CUndosView(CUndosView& other) : CStorageView(new CFlushableStorageKV(other.DB())) {} + explicit CUndosView(CStorageKV& st) : CStorageView(new CFlushableStorageKV(st)) {} + + void ForEachUndo(std::function)> callback, const UndoSourceKey& start = {}); + + [[nodiscard]] std::optional GetUndo(UndoSourceKey const & key) const; + Res SetUndo(const UndoSourceKey& key, const CUndo& undo); + Res DelUndo(const UndoSourceKey & key); + + void AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height); + void OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height); + + // tags + struct ByMultiUndoKey { static constexpr uint8_t prefix() { return 'n'; } }; +}; + +extern std::unique_ptr pundosDB; +extern std::unique_ptr pundosView; #endif //DEFI_MASTERNODES_UNDOS_H diff --git a/src/miner.cpp b/src/miner.cpp index 68d846a408b..4293b5159d7 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -213,6 +213,7 @@ std::unique_ptr BlockAssembler::CreateNewBlock(const CScript& sc int nPackagesSelected = 0; int nDescendantsUpdated = 0; CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); if (!blockTime) { UpdateTime(pblock, consensus, pindexPrev); // update time before tx packaging } @@ -354,7 +355,7 @@ std::unique_ptr BlockAssembler::CreateNewBlock(const CScript& sc && nHeight < chainparams.GetConsensus().EunosKampungHeight) { // includes coinbase account changes ApplyGeneralCoinbaseTx(mnview, *(pblock->vtx[0]), nHeight, nFees, chainparams.GetConsensus()); - pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot()); + pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot(undosView)); } LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", 0.001 * (nTime1 - nTimeStart), nPackagesSelected, nDescendantsUpdated, 0.001 * (nTime2 - nTime1), 0.001 * (nTime2 - nTimeStart)); diff --git a/src/test/setup_common.cpp b/src/test/setup_common.cpp index a29ad05681e..82b148106a5 100644 --- a/src/test/setup_common.cpp +++ b/src/test/setup_common.cpp @@ -113,7 +113,11 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha pcustomcsDB.reset(); pcustomcsDB = std::make_unique(GetDataDir() / "enhancedcs", nMinDbCache << 20, true, true); - pcustomcsview = std::make_unique(*pcustomcsDB.get()); + pcustomcsview = std::make_unique(*pcustomcsDB); + + pundosDB.reset(); + pundosDB = std::make_unique(GetDataDir() / "undos", nMinDbCache << 20, true, true); + pundosView = std::make_unique(*pundosDB); panchorauths.reset(); panchorauths = std::make_unique(); diff --git a/src/test/storage_tests.cpp b/src/test/storage_tests.cpp index a54d8ee2133..746004badba 100644 --- a/src/test/storage_tests.cpp +++ b/src/test/storage_tests.cpp @@ -81,13 +81,18 @@ BOOST_AUTO_TEST_CASE(flushableType) BOOST_AUTO_TEST_CASE(undo) { - CStorageKV & base_raw = pcustomcsview->GetStorage(); + CCustomCSView view(*pcustomcsview); + CUndosView undoView(*pundosView); + auto& base_raw = view.GetStorage(); + auto& undo_raw = undoView.GetStorage(); + auto undoStart = TakeSnapshot(undo_raw); + // place some "old" record - pcustomcsview->Write("testkey1", "value0"); + view.Write("testkey1", "value0"); auto snapStart = TakeSnapshot(base_raw); - CCustomCSView mnview(*pcustomcsview); + auto mnview(view); BOOST_CHECK(mnview.Write("testkey1", "value1")); // modify BOOST_CHECK(mnview.Write("testkey2", "value2")); // insert @@ -107,17 +112,21 @@ BOOST_AUTO_TEST_CASE(undo) BOOST_CHECK(snap1.at(ToBytes("testkey2")) == ToBytes("value2")); // write undo - pcustomcsview->SetUndo(UndoKey{1, uint256S("0x1")}, undo); + auto snap_undo1 = TakeSnapshot(base_raw); + undoView.SetUndo({{1, uint256S("0x1")}, UndoSource::CustomView}, undo); - auto snap2 = TakeSnapshot(base_raw); - BOOST_CHECK(snap2.size() - snap1.size() == 1); // undo - BOOST_CHECK(snap2.size() - snapStart.size() == 2); // onew new record + undo + auto snap_undo = TakeSnapshot(undo_raw); + BOOST_CHECK_EQUAL(snap_undo.size() - undoStart.size(), 1); // undo - pcustomcsview->OnUndoTx(uint256S("0x1"), 2); // fail + auto snap2 = TakeSnapshot(base_raw); + undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x1"), 2); // fail + mnview.Flush(); BOOST_CHECK(snap2 == TakeSnapshot(base_raw)); - pcustomcsview->OnUndoTx(uint256S("0x2"), 1); // fail + undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x2"), 1); // fail + mnview.Flush(); BOOST_CHECK(snap2 == TakeSnapshot(base_raw)); - pcustomcsview->OnUndoTx(uint256S("0x1"), 1); // success + undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x1"), 1); // success + mnview.Flush(); BOOST_CHECK(snapStart == TakeSnapshot(base_raw)); } diff --git a/src/validation.cpp b/src/validation.cpp index 352c73f22d0..ea1df71e979 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1691,7 +1691,7 @@ static bool GetCreationTransactions(const CBlock& block, const uint32_t id, cons /** Undo the effects of this block (with given index) on the UTXO set represented by coins. * When FAILED is returned, view is left in an indeterminate state. */ -DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, std::vector & disconnectedAnchorConfirms) +DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, CUndosView& undosView, std::vector & disconnectedAnchorConfirms) { bool fClean = true; @@ -1712,7 +1712,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI } // special case: possible undo (first) of custom 'complex changes' for the whole block (expired orders and/or prices) - mnview.OnUndoTx(uint256(), (uint32_t) pindex->nHeight); // undo for "zero hash" + undosView.OnUndoTx(UndoSource::CustomView, mnview, {}, static_cast(pindex->nHeight)); pburnHistoryDB->EraseAccountHistoryHeight(pindex->nHeight); if (paccountHistoryDB) { @@ -1834,7 +1834,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI } // process transactions revert for masternodes - mnview.OnUndoTx(tx.GetHash(), (uint32_t) pindex->nHeight); + undosView.OnUndoTx(UndoSource::CustomView, mnview, tx.GetHash(), pindex->nHeight); } // one time downgrade to revert CInterestRateV2 structure @@ -2448,6 +2448,53 @@ bool ApplyGovVars(CCustomCSView& cache, const CBlockIndex& pindex, const std::ma return false; } +static void PruneUndos(CUndosView& view, const int height) { + bool pruneStarted = false; + auto time = GetTimeMillis(); + CUndosView pruned(view); + view.ForEachUndo([&](const UndoSourceKey& key, const CLazySerialize&) { + if (static_cast(key.height) >= height) { // don't erase checkpoint height + return false; + } + if (!pruneStarted) { + pruneStarted = true; + LogPrintf("Pruning undo data prior %d, it can take a while...\n", height); + } + return pruned.DelUndo(key).ok; + }); + if (pruneStarted) { + auto& map = pruned.GetStorage().GetRaw(); + compactBegin = map.begin()->first; + compactEnd = map.rbegin()->first; + pruned.Flush(); + LogPrintf("Pruning undo data finished.\n"); + LogPrint(BCLog::BENCH, " - Pruning undo data takes: %dms\n", GetTimeMillis() - time); + } +} + +static void PruneInterest(CCustomCSView& mnview, const CChainParams& chainparams, int height) { + if (height <= chainparams.GetConsensus().FortCanningHillHeight) { + return; + } + CCustomCSView view(mnview); + mnview.ForEachVaultInterest([&](const CVaultId& vaultId, DCT_ID tokenId, CInterestRate) { + view.EraseBy(std::make_pair(vaultId, tokenId)); + return true; + }); + view.Flush(); +} + +static void PruneData(CCustomCSView& mnview, CUndosView& undosView, const CChainParams& chainparams, int height) { + auto &checkpoints = chainparams.Checkpoints().mapCheckpoints; + auto it = checkpoints.lower_bound(height); + if (it != checkpoints.begin()) { + --it; + PruneUndos(undosView, it->first); + // we can safely delete old interest keys + PruneInterest(mnview, chainparams, height); + } +} + bool StopOrInterruptConnect(const CBlockIndex *pIndex, CValidationState& state) { if (!fStopOrInterrupt) return false; @@ -2476,8 +2523,8 @@ bool StopOrInterruptConnect(const CBlockIndex *pIndex, CValidationState& state) /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock () * can fail if those validity checks fail (among other reasons). */ -bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, CCustomCSView& mnview, const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck) +bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, CUndosView& undosView, + const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck) { AssertLockHeld(cs_main); assert(pindex); @@ -2544,13 +2591,17 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl pcustomcsview->CreateDFIToken(); // init view|db with genesis here for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = *block.vtx[i]; + CCustomCSView mnviewCopy(mnview); CHistoryWriters writers{paccountHistoryDB.get(), nullptr, nullptr}; - const auto res = ApplyCustomTx(mnview, view, *block.vtx[i], chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); + const auto res = ApplyCustomTx(mnviewCopy, view, *block.vtx[i], chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); if (!res.ok) { return error("%s: Genesis block ApplyCustomTx failed. TX: %s Error: %s", - __func__, block.vtx[i]->GetHash().ToString(), res.msg); + __func__, tx.GetHash().ToString(), res.msg); } - AddCoins(view, *block.vtx[i], 0); + undosView.AddUndo(UndoSource::CustomView, mnview, mnviewCopy, tx.GetHash(), pindex->nHeight); + mnviewCopy.Flush(); + AddCoins(view, tx, 0); } } return true; @@ -2813,8 +2864,9 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl __func__, tx.GetHash().ToString(), FormatStateMessage(state)); } + CCustomCSView mnviewCopy(accountsView); CHistoryWriters writers{paccountHistoryDB.get(), pburnHistoryDB.get(), pvaultHistoryDB.get()}; - const auto res = ApplyCustomTx(mnview, view, tx, chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); + const auto res = ApplyCustomTx(mnviewCopy, view, tx, chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); if (!res.ok && (res.code & CustomTxErrCodes::Fatal)) { if (pindex->nHeight >= chainparams.GetConsensus().EunosHeight) { return state.Invalid(ValidationInvalidReason::CONSENSUS, @@ -2826,6 +2878,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl __func__, tx.GetHash().ToString(), res.msg); } } + undosView.AddUndo(UndoSource::CustomView, accountsView, mnviewCopy, tx.GetHash(), pindex->nHeight); + mnviewCopy.Flush(); // log if (!fJustCheck && !res.msg.empty()) { if (res.ok) { @@ -2967,7 +3021,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl && pindex->nHeight < chainparams.GetConsensus().EunosKampungHeight) { bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); - if (block.hashMerkleRoot != Hash2(hashMerkleRoot2, accountsView.MerkleRoot())) { + if (block.hashMerkleRoot != Hash2(hashMerkleRoot2, accountsView.MerkleRoot(undosView))) { return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-txnmrklroot", "hashMerkleRoot mismatch"); } @@ -3039,15 +3093,9 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // Masternode updates ProcessMasternodeUpdates(pindex, cache, view, chainparams); - // construct undo - auto& flushable = cache.GetStorage(); - auto undo = CUndo::Construct(mnview.GetStorage(), flushable.GetRaw()); - // flush changes to underlying view + undosView.AddUndo(UndoSource::CustomView, mnview, cache, {}, pindex->nHeight); + cache.Flush(); - // write undo - if (!undo.before.empty()) { - mnview.SetUndo(UndoKey{static_cast(pindex->nHeight), uint256() }, undo); // "zero hash" - } } // Write any UTXO burns @@ -3068,41 +3116,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl } mnview.SetLastHeight(pindex->nHeight); - auto &checkpoints = chainparams.Checkpoints().mapCheckpoints; - auto it = checkpoints.lower_bound(pindex->nHeight); - if (it != checkpoints.begin()) { - --it; - bool pruneStarted = false; - auto time = GetTimeMillis(); - CCustomCSView pruned(mnview); - mnview.ForEachUndo([&](UndoKey const & key, CLazySerialize) { - if (key.height >= static_cast(it->first)) { // don't erase checkpoint height - return false; - } - if (!pruneStarted) { - pruneStarted = true; - LogPrintf("Pruning undo data prior %d, it can take a while...\n", it->first); - } - return pruned.DelUndo(key).ok; - }); - if (pruneStarted) { - auto& map = pruned.GetStorage().GetRaw(); - compactBegin = map.begin()->first; - compactEnd = map.rbegin()->first; - pruned.Flush(); - LogPrintf("Pruning undo data finished.\n"); - LogPrint(BCLog::BENCH, " - Pruning undo data takes: %dms\n", GetTimeMillis() - time); - } - // we can safety delete old interest keys - if (it->first > chainparams.GetConsensus().FortCanningHillHeight) { - CCustomCSView view(mnview); - mnview.ForEachVaultInterest([&](const CVaultId& vaultId, DCT_ID tokenId, CInterestRate) { - view.EraseBy(std::make_pair(vaultId, tokenId)); - return true; - }); - view.Flush(); - } - } + PruneData(mnview, undosView, chainparams, pindex->nHeight); if (pindex->nHeight >= chainparams.GetConsensus().GreatWorldHeight) { // Remove any TXs from mempool that are now expired @@ -5050,10 +5064,11 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(&CoinsTip()); - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); std::vector disconnectedConfirms; - if (DisconnectBlock(block, pindexDelete, view, mnview, disconnectedConfirms) != DISCONNECT_OK) { + if (DisconnectBlock(block, pindexDelete, view, mnview, undosView, disconnectedConfirms) != DISCONNECT_OK) { // no usable history if (paccountHistoryDB) { paccountHistoryDB->Discard(); @@ -5067,7 +5082,7 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha m_disconnectTip = false; return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); } - bool flushed = view.Flush() && mnview.Flush(); + bool flushed = view.Flush() && mnview.Flush() && undosView.Flush(); assert(flushed); // flush history @@ -5216,9 +5231,10 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); { CCoinsViewCache view(&CoinsTip()); - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); std::vector rewardedAnchors; - bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, mnview, chainparams, rewardedAnchors); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, mnview, undosView, chainparams, rewardedAnchors); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) { @@ -5238,7 +5254,7 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime3 - nTime2) * MILLI, nTimeConnectTotal * MICRO, nTimeConnectTotal * MILLI / nBlocksTotal); - bool flushed = view.Flush() && mnview.Flush(); + bool flushed = view.Flush() && mnview.Flush() && undosView.Flush(); assert(flushed); // flush history @@ -6760,7 +6776,8 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); CCoinsViewCache viewNew(&::ChainstateActive().CoinsTip()); std::vector dummyRewardedAnchors; - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); uint256 block_hash(block.GetHash()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; @@ -6775,7 +6792,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, mnview, chainparams, dummyRewardedAnchors, true)) + if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, mnview, undosView, chainparams, dummyRewardedAnchors, true)) return false; assert(state.IsValid()); @@ -7158,7 +7175,8 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); CCoinsViewCache coins(coinsview); - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; @@ -7203,7 +7221,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); std::vector disconnectedConfirms; // dummy - DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins, mnview, disconnectedConfirms); + DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins, mnview, undosView, disconnectedConfirms); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -7238,7 +7256,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); std::vector dummyRewardedAnchors; - if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, mnview, chainparams, dummyRewardedAnchors)) + if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, mnview, undosView, chainparams, dummyRewardedAnchors)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); if (ShutdownRequested()) return true; } @@ -7274,12 +7292,13 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i return true; } -bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview) +bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview, CUndosView* undosView) { LOCK(cs_main); CCoinsViewCache cache(view); CCustomCSView mncache(*mnview); + CUndosView undosCache(*undosView); std::vector hashHeads = view->GetHeadBlocks(); if (hashHeads.empty()) return true; // We're already in a consistent state. @@ -7318,7 +7337,7 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCu } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); std::vector disconnectedConfirms; // dummy - DisconnectResult res = DisconnectBlock(block, pindexOld, cache, mncache, disconnectedConfirms); + DisconnectResult res = DisconnectBlock(block, pindexOld, cache, mncache, undosCache, disconnectedConfirms); if (res == DISCONNECT_FAILED) { return error("RollbackBlock(): DisconnectBlock failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } @@ -7346,8 +7365,8 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCu return true; } -bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview) { - return ::ChainstateActive().ReplayBlocks(params, view, mnview); +bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview, CUndosView* undosView) { + return ::ChainstateActive().ReplayBlocks(params, view, mnview, undosView); } //! Helper for CChainState::RewindBlockIndex diff --git a/src/validation.h b/src/validation.h index 35fc0b33dfa..b3aeffdf282 100644 --- a/src/validation.h +++ b/src/validation.h @@ -48,6 +48,7 @@ class CConnman; class CScriptCheck; class CBlockPolicyEstimator; class CTxMemPool; +class CUndosView; class CValidationState; struct ChainTxData; @@ -437,7 +438,7 @@ class CVerifyDB { }; /** Replay blocks that aren't fully applied to the database. */ -bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache); +bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache, CUndosView* undosView); CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -725,9 +726,10 @@ class CChainState { bool AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: - DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, std::vector & disconnectedAnchorConfirms); - bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, CCustomCSView& cache, const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, CUndosView& undosView, + std::vector & disconnectedAnchorConfirms); + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, CUndosView& undosView, + const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. bool DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); @@ -737,7 +739,7 @@ class CChainState { bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache); + bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache, CUndosView* undosView); bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); bool LoadGenesisBlock(const CChainParams& chainparams); diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index ed19bc35dcd..b0ffd07835a 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -621,6 +621,7 @@ def cache_path(*paths): # Remove custom dirs shutil.rmtree(cache_path('burn')) + shutil.rmtree(cache_path('undos')) for entry in os.listdir(cache_path()): if entry not in ['chainstate', 'blocks', 'enhancedcs', 'anchors', 'history']: # Only keep chainstate and blocks folder From 6d5d5a95febdd22d1bd9d3a6c261c70410f17272 Mon Sep 17 00:00:00 2001 From: Peter Bushnell Date: Sun, 17 Jul 2022 13:42:59 +0100 Subject: [PATCH 2/2] Revert "Migrate undos to dedicated DB (#1381)" This reverts commit 67e12c85d401aef58b846bdc4dd318166213099d. --- src/flushablestorage.h | 3 - src/init.cpp | 55 +------ src/masternodes/masternodes.cpp | 38 +++-- src/masternodes/masternodes.h | 14 +- src/masternodes/mn_checks.cpp | 8 + src/masternodes/undo.h | 17 -- src/masternodes/undos.cpp | 48 +----- src/masternodes/undos.h | 27 +-- src/miner.cpp | 3 +- src/test/setup_common.cpp | 6 +- src/test/storage_tests.cpp | 29 ++-- src/validation.cpp | 155 ++++++++---------- src/validation.h | 12 +- .../test_framework/test_framework.py | 1 - 14 files changed, 139 insertions(+), 277 deletions(-) diff --git a/src/flushablestorage.h b/src/flushablestorage.h index 3d50ea0c85b..fd41d96a0d9 100644 --- a/src/flushablestorage.h +++ b/src/flushablestorage.h @@ -503,9 +503,6 @@ class CStorageView { } } } - CFlushableStorageKV& GetStorage() { - return static_cast(DB()); - } virtual bool Flush() { return DB().Flush(); } void Discard() { DB().Discard(); } diff --git a/src/init.cpp b/src/init.cpp index 2d294623622..ca0518ee6e4 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1340,48 +1340,6 @@ void SetupAnchorSPVDatabases(bool resync) { } } - -void MigrateDBs() -{ - auto it = pundosView->LowerBound(UndoSourceKey{}); - if (it.Valid()) { - return; - } - - auto time = GetTimeMillis(); - - // Migrate Undos - auto migrationStarted{false}; - auto mnview(*pcustomcsview); - pcustomcsview->ForEachUndo([&](const UndoKey& key, const CUndo& undo){ - if (!migrationStarted) { - migrationStarted = true; - LogPrintf("Migrating undo entries, might take a while...\n"); - } - - pundosView->SetUndo({{key.height, key.txid}, UndoSource::CustomView}, undo); - mnview.DelUndo(key); - - return true; - }); - - if (migrationStarted) { - pundosView->Flush(); - pundosDB->Flush(); - - auto& map = mnview.GetStorage().GetRaw(); - const auto compactBegin = map.begin()->first; - const auto compactEnd = map.rbegin()->first; - mnview.Flush(); - pcustomcsview->Flush(); - pcustomcsDB->Flush(); - pcustomcsDB->Compact(compactBegin, compactEnd); - - LogPrintf("Migrating undo data finished.\n"); - LogPrint(BCLog::BENCH, " - Migrating undo data takes: %dms\n", GetTimeMillis() - time); - } -} - bool SetupInterruptArg(const std::string &argName, std::string &hashStore, int &heightStore) { // Experimental: Block height or hash to invalidate on and stop sync auto val = gArgs.GetArg(argName, ""); @@ -1759,7 +1717,7 @@ bool AppInitMain(InitInterfaces& interfaces) pcustomcsDB.reset(); pcustomcsDB = std::make_unique(GetDataDir() / "enhancedcs", nCustomCacheSize, false, fReset || fReindexChainState); pcustomcsview.reset(); - pcustomcsview = std::make_unique(*pcustomcsDB); + pcustomcsview = std::make_unique(*pcustomcsDB.get()); if (!fReset && !fReindexChainState) { if (!pcustomcsDB->IsEmpty() && pcustomcsview->GetDbVersion() != CCustomCSView::DbVersion) { @@ -1791,15 +1749,6 @@ bool AppInitMain(InitInterfaces& interfaces) pvaultHistoryDB = std::make_unique(GetDataDir() / "vault", nCustomCacheSize, false, fReset || fReindexChainState); } - // Create Undo DB - pundosDB.reset(); - pundosDB = std::make_unique(GetDataDir() / "undos", nCustomCacheSize, false, fReset || fReindexChainState); - pundosView.reset(); - pundosView = std::make_unique(*pundosDB); - - // Migrate FutureSwaps and Undos to their own new DBs. - MigrateDBs(); - // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!::ChainstateActive().CoinsDB().Upgrade()) { @@ -1808,7 +1757,7 @@ bool AppInitMain(InitInterfaces& interfaces) } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get(), pundosView.get())) { + if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get())) { strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; break; } diff --git a/src/masternodes/masternodes.cpp b/src/masternodes/masternodes.cpp index 915a2df6231..1ca14e3a589 100644 --- a/src/masternodes/masternodes.cpp +++ b/src/masternodes/masternodes.cpp @@ -709,10 +709,7 @@ void CSettingsView::SetDexStatsLastHeight(const int32_t height) std::optional CSettingsView::GetDexStatsLastHeight() { - if (auto res = ReadBy(DEX_STATS_LAST_HEIGHT)) { - return res; - } - return {}; + return ReadBy(DEX_STATS_LAST_HEIGHT); } void CSettingsView::SetDexStatsEnabled(const bool enabled) @@ -881,6 +878,16 @@ void CCustomCSView::CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::Ancho } } +void CCustomCSView::OnUndoTx(uint256 const & txid, uint32_t height) +{ + const auto undo = GetUndo(UndoKey{height, txid}); + if (!undo) { + return; // not custom tx, or no changes done + } + CUndo::Revert(GetStorage(), *undo); // revert the changes of this tx + DelUndo(UndoKey{height, txid}); // erase undo data, it served its purpose +} + bool CCustomCSView::CanSpend(const uint256 & txId, int height) const { auto node = GetMasternode(txId); @@ -1078,31 +1085,28 @@ Res CCustomCSView::PopulateCollateralData(CCollateralLoans& result, CVaultId con return Res::Ok(); } -uint256 CCustomCSView::MerkleRoot(CUndosView& undo) { +uint256 CCustomCSView::MerkleRoot() { auto rawMap = GetStorage().GetRaw(); if (rawMap.empty()) { return {}; } auto isAttributes = [](const TBytes& key) { + MapKV map = {std::make_pair(key, TBytes{})}; // Attributes should not be part of merkle root static const std::string attributes("ATTRIBUTES"); - MapKV map = {std::make_pair(key, TBytes{})}; auto it = NewKVIterator(attributes, map); return it.Valid() && it.Key() == attributes; }; - auto& undoStorage = undo.GetStorage(); - auto& undoMap = undoStorage.GetRaw(); - auto it = NewKVIterator(UndoSourceKey{}, undoMap); + + auto it = NewKVIterator(UndoKey{}, rawMap); for (; it.Valid(); it.Next()) { - if (it.Key().key == UndoSource::CustomView) { - CUndo value = it.Value(); - auto& map = value.before; - for (auto it = map.begin(); it != map.end();) { - isAttributes(it->first) ? map.erase(it++) : ++it; - } - auto key = std::make_pair(CUndosBaseView::ByUndoKey::prefix(), static_cast(it.Key())); - rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value); + CUndo value = it.Value(); + auto& map = value.before; + for (auto it = map.begin(); it != map.end();) { + isAttributes(it->first) ? map.erase(it++) : ++it; } + auto key = std::make_pair(CUndosView::ByUndoKey::prefix(), static_cast(it.Key())); + rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value); } std::vector hashes; diff --git a/src/masternodes/masternodes.h b/src/masternodes/masternodes.h index 6f6ea335e20..a9db52b6e02 100644 --- a/src/masternodes/masternodes.h +++ b/src/masternodes/masternodes.h @@ -390,7 +390,7 @@ class CCustomCSView , public CTokensView , public CAccountsView , public CCommunityBalancesView - , public CUndosBaseView + , public CUndosView , public CPoolPairView , public CGovView , public CAnchorConfirmsView @@ -411,7 +411,7 @@ class CCustomCSView CTokensView :: ID, Symbol, CreationTx, LastDctId, CAccountsView :: ByBalanceKey, ByHeightKey, ByFuturesSwapKey, CCommunityBalancesView :: ById, - CUndosBaseView :: ByUndoKey, + CUndosView :: ByUndoKey, CPoolPairView :: ByID, ByPair, ByShare, ByIDPair, ByPoolSwap, ByReserves, ByRewardPct, ByRewardLoanPct, ByPoolReward, ByDailyReward, ByCustomReward, ByTotalLiquidity, ByDailyLoanReward, ByPoolLoanReward, ByTokenDexFeePct, @@ -461,6 +461,9 @@ class CCustomCSView /// @todo newbase move to networking? void CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::AnchorRec* anchor, const uint256 & btcTxHash, const CKey &masternodeKey); + // simplified version of undo, without any unnecessary undo data + void OnUndoTx(uint256 const & txid, uint32_t height); + bool CanSpend(const uint256 & txId, int height) const; bool CalculateOwnerRewards(CScript const & owner, uint32_t height); @@ -484,7 +487,12 @@ class CCustomCSView void SetGlobalCustomTxExpiration(const uint32_t height); uint32_t GetGlobalCustomTxExpiration() const; - uint256 MerkleRoot(CUndosView& undo); + uint256 MerkleRoot(); + + // we construct it as it + CFlushableStorageKV& GetStorage() { + return static_cast(DB()); + } virtual CAccountHistoryStorage* GetAccountHistoryStore(); CVaultHistoryStorage* GetVaultHistoryStore(); diff --git a/src/masternodes/mn_checks.cpp b/src/masternodes/mn_checks.cpp index d0b447db2dd..9589dd6a9d6 100644 --- a/src/masternodes/mn_checks.cpp +++ b/src/masternodes/mn_checks.cpp @@ -3774,7 +3774,15 @@ Res ApplyCustomTx(CCustomCSView& mnview, const CCoinsViewCache& coins, const CTr return res; } + // construct undo + auto& flushable = view.GetStorage(); + auto undo = CUndo::Construct(mnview.GetStorage(), flushable.GetRaw()); + // flush changes view.Flush(); + // write undo + if (!undo.before.empty()) { + mnview.SetUndo(UndoKey{height, tx.GetHash()}, undo); + } return res; } diff --git a/src/masternodes/undo.h b/src/masternodes/undo.h index 27c5d596769..b9f6e1e967c 100644 --- a/src/masternodes/undo.h +++ b/src/masternodes/undo.h @@ -12,11 +12,6 @@ #include #include -// Enum for future support of multiple sources of undo data, not just CustomView. -enum UndoSource : uint8_t { - CustomView = 0, -}; - struct UndoKey { uint32_t height; // height is there to be able to prune older undos using lexicographic iteration uint256 txid; @@ -30,18 +25,6 @@ struct UndoKey { } }; -struct UndoSourceKey : UndoKey { - uint8_t key; - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(UndoKey, *this); - READWRITE(key); - } -}; - struct CUndo { MapKV before; diff --git a/src/masternodes/undos.cpp b/src/masternodes/undos.cpp index 40a12efe81c..47bd1d9bfdb 100644 --- a/src/masternodes/undos.cpp +++ b/src/masternodes/undos.cpp @@ -4,61 +4,29 @@ #include -void CUndosBaseView::ForEachUndo(std::function)> callback, UndoKey const & start) +void CUndosView::ForEachUndo(std::function)> callback, UndoKey const & start) { ForEach(callback, start); } -Res CUndosBaseView::DelUndo(UndoKey const & key) +Res CUndosView::SetUndo(UndoKey const & key, CUndo const & undo) { - EraseBy(key); + WriteBy(key, undo); return Res::Ok(); } -void CUndosView::ForEachUndo(std::function)> callback, const UndoSourceKey& start) +Res CUndosView::DelUndo(UndoKey const & key) { - ForEach(callback, start); -} - -void CUndosView::AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height) -{ - auto& flushable = cache.GetStorage(); - auto& rawMap = flushable.GetRaw(); - if (!rawMap.empty()) { - SetUndo({{height, txid}, key}, CUndo::Construct(source.GetStorage(), rawMap)); - } -} - -Res CUndosView::SetUndo(UndoSourceKey const & key, CUndo const & undo) -{ - WriteBy(key, undo); + EraseBy(key); return Res::Ok(); } -void CUndosView::OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height) -{ - const auto undo = GetUndo({{height, txid}, key}); - if (!undo) { - return; // not custom tx, or no changes done - } - CUndo::Revert(source.GetStorage(), *undo); // revert the changes of this tx - DelUndo({{height, txid}, key}); // erase undo data, it served its purpose -} - -std::optional CUndosView::GetUndo(UndoSourceKey const & key) const +std::optional CUndosView::GetUndo(UndoKey const & key) const { CUndo val; - if (ReadBy(key, val)) { + bool ok = ReadBy(key, val); + if (ok) { return val; } return {}; } - -Res CUndosView::DelUndo(const UndoSourceKey & key) -{ - EraseBy(key); - return Res::Ok(); -} - -std::unique_ptr pundosDB; -std::unique_ptr pundosView; diff --git a/src/masternodes/undos.h b/src/masternodes/undos.h index dbb3b19ba45..bf9ebe1f1cd 100644 --- a/src/masternodes/undos.h +++ b/src/masternodes/undos.h @@ -9,36 +9,17 @@ #include #include -class CUndosBaseView : public virtual CStorageView { +class CUndosView : public virtual CStorageView { public: void ForEachUndo(std::function)> callback, UndoKey const & start = {}); - Res DelUndo(const UndoKey & key); + std::optional GetUndo(UndoKey const & key) const; + Res SetUndo(UndoKey const & key, CUndo const & undo); + Res DelUndo(UndoKey const & key); // tags struct ByUndoKey { static constexpr uint8_t prefix() { return 'u'; } }; }; -class CUndosView : public CStorageView -{ -public: - CUndosView(CUndosView& other) : CStorageView(new CFlushableStorageKV(other.DB())) {} - explicit CUndosView(CStorageKV& st) : CStorageView(new CFlushableStorageKV(st)) {} - - void ForEachUndo(std::function)> callback, const UndoSourceKey& start = {}); - - [[nodiscard]] std::optional GetUndo(UndoSourceKey const & key) const; - Res SetUndo(const UndoSourceKey& key, const CUndo& undo); - Res DelUndo(const UndoSourceKey & key); - - void AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height); - void OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height); - - // tags - struct ByMultiUndoKey { static constexpr uint8_t prefix() { return 'n'; } }; -}; - -extern std::unique_ptr pundosDB; -extern std::unique_ptr pundosView; #endif //DEFI_MASTERNODES_UNDOS_H diff --git a/src/miner.cpp b/src/miner.cpp index 4293b5159d7..68d846a408b 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -213,7 +213,6 @@ std::unique_ptr BlockAssembler::CreateNewBlock(const CScript& sc int nPackagesSelected = 0; int nDescendantsUpdated = 0; CCustomCSView mnview(*pcustomcsview); - CUndosView undosView(*pundosView); if (!blockTime) { UpdateTime(pblock, consensus, pindexPrev); // update time before tx packaging } @@ -355,7 +354,7 @@ std::unique_ptr BlockAssembler::CreateNewBlock(const CScript& sc && nHeight < chainparams.GetConsensus().EunosKampungHeight) { // includes coinbase account changes ApplyGeneralCoinbaseTx(mnview, *(pblock->vtx[0]), nHeight, nFees, chainparams.GetConsensus()); - pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot(undosView)); + pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot()); } LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", 0.001 * (nTime1 - nTimeStart), nPackagesSelected, nDescendantsUpdated, 0.001 * (nTime2 - nTime1), 0.001 * (nTime2 - nTimeStart)); diff --git a/src/test/setup_common.cpp b/src/test/setup_common.cpp index 82b148106a5..a29ad05681e 100644 --- a/src/test/setup_common.cpp +++ b/src/test/setup_common.cpp @@ -113,11 +113,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha pcustomcsDB.reset(); pcustomcsDB = std::make_unique(GetDataDir() / "enhancedcs", nMinDbCache << 20, true, true); - pcustomcsview = std::make_unique(*pcustomcsDB); - - pundosDB.reset(); - pundosDB = std::make_unique(GetDataDir() / "undos", nMinDbCache << 20, true, true); - pundosView = std::make_unique(*pundosDB); + pcustomcsview = std::make_unique(*pcustomcsDB.get()); panchorauths.reset(); panchorauths = std::make_unique(); diff --git a/src/test/storage_tests.cpp b/src/test/storage_tests.cpp index 746004badba..a54d8ee2133 100644 --- a/src/test/storage_tests.cpp +++ b/src/test/storage_tests.cpp @@ -81,18 +81,13 @@ BOOST_AUTO_TEST_CASE(flushableType) BOOST_AUTO_TEST_CASE(undo) { - CCustomCSView view(*pcustomcsview); - CUndosView undoView(*pundosView); - auto& base_raw = view.GetStorage(); - auto& undo_raw = undoView.GetStorage(); - auto undoStart = TakeSnapshot(undo_raw); - + CStorageKV & base_raw = pcustomcsview->GetStorage(); // place some "old" record - view.Write("testkey1", "value0"); + pcustomcsview->Write("testkey1", "value0"); auto snapStart = TakeSnapshot(base_raw); - auto mnview(view); + CCustomCSView mnview(*pcustomcsview); BOOST_CHECK(mnview.Write("testkey1", "value1")); // modify BOOST_CHECK(mnview.Write("testkey2", "value2")); // insert @@ -112,21 +107,17 @@ BOOST_AUTO_TEST_CASE(undo) BOOST_CHECK(snap1.at(ToBytes("testkey2")) == ToBytes("value2")); // write undo - auto snap_undo1 = TakeSnapshot(base_raw); - undoView.SetUndo({{1, uint256S("0x1")}, UndoSource::CustomView}, undo); - - auto snap_undo = TakeSnapshot(undo_raw); - BOOST_CHECK_EQUAL(snap_undo.size() - undoStart.size(), 1); // undo + pcustomcsview->SetUndo(UndoKey{1, uint256S("0x1")}, undo); auto snap2 = TakeSnapshot(base_raw); - undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x1"), 2); // fail - mnview.Flush(); + BOOST_CHECK(snap2.size() - snap1.size() == 1); // undo + BOOST_CHECK(snap2.size() - snapStart.size() == 2); // onew new record + undo + + pcustomcsview->OnUndoTx(uint256S("0x1"), 2); // fail BOOST_CHECK(snap2 == TakeSnapshot(base_raw)); - undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x2"), 1); // fail - mnview.Flush(); + pcustomcsview->OnUndoTx(uint256S("0x2"), 1); // fail BOOST_CHECK(snap2 == TakeSnapshot(base_raw)); - undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x1"), 1); // success - mnview.Flush(); + pcustomcsview->OnUndoTx(uint256S("0x1"), 1); // success BOOST_CHECK(snapStart == TakeSnapshot(base_raw)); } diff --git a/src/validation.cpp b/src/validation.cpp index ea1df71e979..352c73f22d0 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1691,7 +1691,7 @@ static bool GetCreationTransactions(const CBlock& block, const uint32_t id, cons /** Undo the effects of this block (with given index) on the UTXO set represented by coins. * When FAILED is returned, view is left in an indeterminate state. */ -DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, CUndosView& undosView, std::vector & disconnectedAnchorConfirms) +DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, std::vector & disconnectedAnchorConfirms) { bool fClean = true; @@ -1712,7 +1712,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI } // special case: possible undo (first) of custom 'complex changes' for the whole block (expired orders and/or prices) - undosView.OnUndoTx(UndoSource::CustomView, mnview, {}, static_cast(pindex->nHeight)); + mnview.OnUndoTx(uint256(), (uint32_t) pindex->nHeight); // undo for "zero hash" pburnHistoryDB->EraseAccountHistoryHeight(pindex->nHeight); if (paccountHistoryDB) { @@ -1834,7 +1834,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI } // process transactions revert for masternodes - undosView.OnUndoTx(UndoSource::CustomView, mnview, tx.GetHash(), pindex->nHeight); + mnview.OnUndoTx(tx.GetHash(), (uint32_t) pindex->nHeight); } // one time downgrade to revert CInterestRateV2 structure @@ -2448,53 +2448,6 @@ bool ApplyGovVars(CCustomCSView& cache, const CBlockIndex& pindex, const std::ma return false; } -static void PruneUndos(CUndosView& view, const int height) { - bool pruneStarted = false; - auto time = GetTimeMillis(); - CUndosView pruned(view); - view.ForEachUndo([&](const UndoSourceKey& key, const CLazySerialize&) { - if (static_cast(key.height) >= height) { // don't erase checkpoint height - return false; - } - if (!pruneStarted) { - pruneStarted = true; - LogPrintf("Pruning undo data prior %d, it can take a while...\n", height); - } - return pruned.DelUndo(key).ok; - }); - if (pruneStarted) { - auto& map = pruned.GetStorage().GetRaw(); - compactBegin = map.begin()->first; - compactEnd = map.rbegin()->first; - pruned.Flush(); - LogPrintf("Pruning undo data finished.\n"); - LogPrint(BCLog::BENCH, " - Pruning undo data takes: %dms\n", GetTimeMillis() - time); - } -} - -static void PruneInterest(CCustomCSView& mnview, const CChainParams& chainparams, int height) { - if (height <= chainparams.GetConsensus().FortCanningHillHeight) { - return; - } - CCustomCSView view(mnview); - mnview.ForEachVaultInterest([&](const CVaultId& vaultId, DCT_ID tokenId, CInterestRate) { - view.EraseBy(std::make_pair(vaultId, tokenId)); - return true; - }); - view.Flush(); -} - -static void PruneData(CCustomCSView& mnview, CUndosView& undosView, const CChainParams& chainparams, int height) { - auto &checkpoints = chainparams.Checkpoints().mapCheckpoints; - auto it = checkpoints.lower_bound(height); - if (it != checkpoints.begin()) { - --it; - PruneUndos(undosView, it->first); - // we can safely delete old interest keys - PruneInterest(mnview, chainparams, height); - } -} - bool StopOrInterruptConnect(const CBlockIndex *pIndex, CValidationState& state) { if (!fStopOrInterrupt) return false; @@ -2523,8 +2476,8 @@ bool StopOrInterruptConnect(const CBlockIndex *pIndex, CValidationState& state) /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock () * can fail if those validity checks fail (among other reasons). */ -bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, CUndosView& undosView, - const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck) +bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, CCustomCSView& mnview, const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck) { AssertLockHeld(cs_main); assert(pindex); @@ -2591,17 +2544,13 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl pcustomcsview->CreateDFIToken(); // init view|db with genesis here for (size_t i = 0; i < block.vtx.size(); ++i) { - const auto& tx = *block.vtx[i]; - CCustomCSView mnviewCopy(mnview); CHistoryWriters writers{paccountHistoryDB.get(), nullptr, nullptr}; - const auto res = ApplyCustomTx(mnviewCopy, view, *block.vtx[i], chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); + const auto res = ApplyCustomTx(mnview, view, *block.vtx[i], chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); if (!res.ok) { return error("%s: Genesis block ApplyCustomTx failed. TX: %s Error: %s", - __func__, tx.GetHash().ToString(), res.msg); + __func__, block.vtx[i]->GetHash().ToString(), res.msg); } - undosView.AddUndo(UndoSource::CustomView, mnview, mnviewCopy, tx.GetHash(), pindex->nHeight); - mnviewCopy.Flush(); - AddCoins(view, tx, 0); + AddCoins(view, *block.vtx[i], 0); } } return true; @@ -2864,9 +2813,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl __func__, tx.GetHash().ToString(), FormatStateMessage(state)); } - CCustomCSView mnviewCopy(accountsView); CHistoryWriters writers{paccountHistoryDB.get(), pburnHistoryDB.get(), pvaultHistoryDB.get()}; - const auto res = ApplyCustomTx(mnviewCopy, view, tx, chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); + const auto res = ApplyCustomTx(mnview, view, tx, chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); if (!res.ok && (res.code & CustomTxErrCodes::Fatal)) { if (pindex->nHeight >= chainparams.GetConsensus().EunosHeight) { return state.Invalid(ValidationInvalidReason::CONSENSUS, @@ -2878,8 +2826,6 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl __func__, tx.GetHash().ToString(), res.msg); } } - undosView.AddUndo(UndoSource::CustomView, accountsView, mnviewCopy, tx.GetHash(), pindex->nHeight); - mnviewCopy.Flush(); // log if (!fJustCheck && !res.msg.empty()) { if (res.ok) { @@ -3021,7 +2967,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl && pindex->nHeight < chainparams.GetConsensus().EunosKampungHeight) { bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); - if (block.hashMerkleRoot != Hash2(hashMerkleRoot2, accountsView.MerkleRoot(undosView))) { + if (block.hashMerkleRoot != Hash2(hashMerkleRoot2, accountsView.MerkleRoot())) { return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-txnmrklroot", "hashMerkleRoot mismatch"); } @@ -3093,9 +3039,15 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // Masternode updates ProcessMasternodeUpdates(pindex, cache, view, chainparams); - undosView.AddUndo(UndoSource::CustomView, mnview, cache, {}, pindex->nHeight); - + // construct undo + auto& flushable = cache.GetStorage(); + auto undo = CUndo::Construct(mnview.GetStorage(), flushable.GetRaw()); + // flush changes to underlying view cache.Flush(); + // write undo + if (!undo.before.empty()) { + mnview.SetUndo(UndoKey{static_cast(pindex->nHeight), uint256() }, undo); // "zero hash" + } } // Write any UTXO burns @@ -3116,7 +3068,41 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl } mnview.SetLastHeight(pindex->nHeight); - PruneData(mnview, undosView, chainparams, pindex->nHeight); + auto &checkpoints = chainparams.Checkpoints().mapCheckpoints; + auto it = checkpoints.lower_bound(pindex->nHeight); + if (it != checkpoints.begin()) { + --it; + bool pruneStarted = false; + auto time = GetTimeMillis(); + CCustomCSView pruned(mnview); + mnview.ForEachUndo([&](UndoKey const & key, CLazySerialize) { + if (key.height >= static_cast(it->first)) { // don't erase checkpoint height + return false; + } + if (!pruneStarted) { + pruneStarted = true; + LogPrintf("Pruning undo data prior %d, it can take a while...\n", it->first); + } + return pruned.DelUndo(key).ok; + }); + if (pruneStarted) { + auto& map = pruned.GetStorage().GetRaw(); + compactBegin = map.begin()->first; + compactEnd = map.rbegin()->first; + pruned.Flush(); + LogPrintf("Pruning undo data finished.\n"); + LogPrint(BCLog::BENCH, " - Pruning undo data takes: %dms\n", GetTimeMillis() - time); + } + // we can safety delete old interest keys + if (it->first > chainparams.GetConsensus().FortCanningHillHeight) { + CCustomCSView view(mnview); + mnview.ForEachVaultInterest([&](const CVaultId& vaultId, DCT_ID tokenId, CInterestRate) { + view.EraseBy(std::make_pair(vaultId, tokenId)); + return true; + }); + view.Flush(); + } + } if (pindex->nHeight >= chainparams.GetConsensus().GreatWorldHeight) { // Remove any TXs from mempool that are now expired @@ -5064,11 +5050,10 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(&CoinsTip()); - CCustomCSView mnview(*pcustomcsview); - CUndosView undosView(*pundosView); + CCustomCSView mnview(*pcustomcsview.get()); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); std::vector disconnectedConfirms; - if (DisconnectBlock(block, pindexDelete, view, mnview, undosView, disconnectedConfirms) != DISCONNECT_OK) { + if (DisconnectBlock(block, pindexDelete, view, mnview, disconnectedConfirms) != DISCONNECT_OK) { // no usable history if (paccountHistoryDB) { paccountHistoryDB->Discard(); @@ -5082,7 +5067,7 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha m_disconnectTip = false; return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); } - bool flushed = view.Flush() && mnview.Flush() && undosView.Flush(); + bool flushed = view.Flush() && mnview.Flush(); assert(flushed); // flush history @@ -5231,10 +5216,9 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); { CCoinsViewCache view(&CoinsTip()); - CCustomCSView mnview(*pcustomcsview); - CUndosView undosView(*pundosView); + CCustomCSView mnview(*pcustomcsview.get()); std::vector rewardedAnchors; - bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, mnview, undosView, chainparams, rewardedAnchors); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, mnview, chainparams, rewardedAnchors); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) { @@ -5254,7 +5238,7 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime3 - nTime2) * MILLI, nTimeConnectTotal * MICRO, nTimeConnectTotal * MILLI / nBlocksTotal); - bool flushed = view.Flush() && mnview.Flush() && undosView.Flush(); + bool flushed = view.Flush() && mnview.Flush(); assert(flushed); // flush history @@ -6776,8 +6760,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); CCoinsViewCache viewNew(&::ChainstateActive().CoinsTip()); std::vector dummyRewardedAnchors; - CCustomCSView mnview(*pcustomcsview); - CUndosView undosView(*pundosView); + CCustomCSView mnview(*pcustomcsview.get()); uint256 block_hash(block.GetHash()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; @@ -6792,7 +6775,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, mnview, undosView, chainparams, dummyRewardedAnchors, true)) + if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, mnview, chainparams, dummyRewardedAnchors, true)) return false; assert(state.IsValid()); @@ -7175,8 +7158,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); CCoinsViewCache coins(coinsview); - CCustomCSView mnview(*pcustomcsview); - CUndosView undosView(*pundosView); + CCustomCSView mnview(*pcustomcsview.get()); CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; @@ -7221,7 +7203,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); std::vector disconnectedConfirms; // dummy - DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins, mnview, undosView, disconnectedConfirms); + DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins, mnview, disconnectedConfirms); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -7256,7 +7238,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); std::vector dummyRewardedAnchors; - if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, mnview, undosView, chainparams, dummyRewardedAnchors)) + if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, mnview, chainparams, dummyRewardedAnchors)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); if (ShutdownRequested()) return true; } @@ -7292,13 +7274,12 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i return true; } -bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview, CUndosView* undosView) +bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview) { LOCK(cs_main); CCoinsViewCache cache(view); CCustomCSView mncache(*mnview); - CUndosView undosCache(*undosView); std::vector hashHeads = view->GetHeadBlocks(); if (hashHeads.empty()) return true; // We're already in a consistent state. @@ -7337,7 +7318,7 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCu } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); std::vector disconnectedConfirms; // dummy - DisconnectResult res = DisconnectBlock(block, pindexOld, cache, mncache, undosCache, disconnectedConfirms); + DisconnectResult res = DisconnectBlock(block, pindexOld, cache, mncache, disconnectedConfirms); if (res == DISCONNECT_FAILED) { return error("RollbackBlock(): DisconnectBlock failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } @@ -7365,8 +7346,8 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCu return true; } -bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview, CUndosView* undosView) { - return ::ChainstateActive().ReplayBlocks(params, view, mnview, undosView); +bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview) { + return ::ChainstateActive().ReplayBlocks(params, view, mnview); } //! Helper for CChainState::RewindBlockIndex diff --git a/src/validation.h b/src/validation.h index b3aeffdf282..35fc0b33dfa 100644 --- a/src/validation.h +++ b/src/validation.h @@ -48,7 +48,6 @@ class CConnman; class CScriptCheck; class CBlockPolicyEstimator; class CTxMemPool; -class CUndosView; class CValidationState; struct ChainTxData; @@ -438,7 +437,7 @@ class CVerifyDB { }; /** Replay blocks that aren't fully applied to the database. */ -bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache, CUndosView* undosView); +bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache); CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -726,10 +725,9 @@ class CChainState { bool AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: - DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, CUndosView& undosView, - std::vector & disconnectedAnchorConfirms); - bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, CUndosView& undosView, - const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, std::vector & disconnectedAnchorConfirms); + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, CCustomCSView& cache, const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. bool DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); @@ -739,7 +737,7 @@ class CChainState { bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache, CUndosView* undosView); + bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache); bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); bool LoadGenesisBlock(const CChainParams& chainparams); diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index b0ffd07835a..ed19bc35dcd 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -621,7 +621,6 @@ def cache_path(*paths): # Remove custom dirs shutil.rmtree(cache_path('burn')) - shutil.rmtree(cache_path('undos')) for entry in os.listdir(cache_path()): if entry not in ['chainstate', 'blocks', 'enhancedcs', 'anchors', 'history']: # Only keep chainstate and blocks folder