From dfe129687f3bd029d465b4a6a9284e7ede60ea38 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Thu, 3 Dec 2020 10:20:23 +0200 Subject: [PATCH 1/9] Better handling of account history Signed-off-by: Anthony Fieroni --- src/flushablestorage.h | 3 +- src/masternodes/accounts.h | 5 +-- src/masternodes/accountshistory.cpp | 47 +++------------------------- src/masternodes/accountshistory.h | 5 ++- src/masternodes/masternodes.cpp | 33 ++++++++++++++++++++ src/masternodes/masternodes.h | 17 ++++++++++ src/masternodes/mn_checks.cpp | 15 ++++++--- src/masternodes/mn_rpc.cpp | 48 ++++++++++++++--------------- src/validation.cpp | 3 +- 9 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/flushablestorage.h b/src/flushablestorage.h index 406021b8e2..a68b0a2e2f 100644 --- a/src/flushablestorage.h +++ b/src/flushablestorage.h @@ -315,8 +315,9 @@ class CFlushableStorageKV : public CStorageKV { class CStorageView { public: + CStorageView() = default; CStorageView(CStorageKV * st) : storage(st) {} - CStorageView() {} + virtual ~CStorageView() = default; template bool Exists(const KeyType& key) const { diff --git a/src/masternodes/accounts.h b/src/masternodes/accounts.h index b575976903..d966fd2ce6 100644 --- a/src/masternodes/accounts.h +++ b/src/masternodes/accounts.h @@ -17,9 +17,10 @@ class CAccountsView : public virtual CStorageView void ForEachBalance(std::function callback, BalanceKey start = {}) const; CTokenAmount GetBalance(CScript const & owner, DCT_ID tokenID) const; - Res AddBalance(CScript const & owner, CTokenAmount amount); + virtual Res AddBalance(CScript const & owner, CTokenAmount amount); + virtual Res SubBalance(CScript const & owner, CTokenAmount amount); + Res AddBalances(CScript const & owner, CBalances const & balances); - Res SubBalance(CScript const & owner, CTokenAmount amount); Res SubBalances(CScript const & owner, CBalances const & balances); // tags diff --git a/src/masternodes/accountshistory.cpp b/src/masternodes/accountshistory.cpp index 849a5dbecf..fb54c00f7c 100644 --- a/src/masternodes/accountshistory.cpp +++ b/src/masternodes/accountshistory.cpp @@ -11,52 +11,15 @@ /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!! const unsigned char CAccountsHistoryView::ByAccountHistoryKey::prefix = 'h'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB -void CAccountsHistoryView::ForEachAccountHistory(std::function callback, AccountHistoryKey start) const +void CAccountsHistoryView::ForEachAccountHistory(std::function callback, AccountHistoryKey start) const { - ForEach([&callback] (AccountHistoryKey const & key, AccountHistoryValue const & val) { - return callback(key.owner,key.blockHeight, key.txn, val.txid, val.category, val.diff); + ForEach([&callback] (AccountHistoryKey const & key, AccountHistoryValue const & value) { + return callback(key, value); }, start); } -Res CAccountsHistoryView::SetAccountHistory(const CScript & owner, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category, TAmounts const & diff) +Res CAccountsHistoryView::SetAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value) { -//// left for debug: -// std::string ownerStr; -// CTxDestination dest; -// if (!ExtractDestination(owner, dest)) { -// ownerStr = owner.GetHex(); -// } else -// ownerStr = EncodeDestination(dest); -// LogPrintf("DEBUG: SetAccountHistory: owner: %s, ownerStr: %s, block: %i, txn: %d, txid: %s, diffs: %ld\n", owner.GetHex().c_str(), ownerStr.c_str(), height, txn, txid.ToString().c_str(), diff.size()); - - WriteBy(AccountHistoryKey{owner, height, txn}, AccountHistoryValue{txid, category, diff}); + WriteBy(key, value); return Res::Ok(); } - -bool CAccountsHistoryView::TrackAffectedAccounts(CStorageKV const & before, MapKV const & diff, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category) { - // txn set to max if called from CreateNewBlock to check account balances, do not track here. - if (!gArgs.GetBoolArg("-acindex", false) || txn == std::numeric_limits::max()) - return false; - - std::map balancesDiff; - using TKey = std::pair; - - for (auto it = diff.lower_bound({CAccountsView::ByBalanceKey::prefix}); it != diff.end() && it->first.at(0) == CAccountsView::ByBalanceKey::prefix; ++it) { - CAmount oldAmount = 0, newAmount = 0; - - if (it->second) { - BytesToDbType(*it->second, newAmount); - } - TBytes beforeVal; - if (before.Read(it->first, beforeVal)) { - BytesToDbType(beforeVal, oldAmount); - } - TKey balanceKey; - BytesToDbType(it->first, balanceKey); - balancesDiff[balanceKey.second.owner][balanceKey.second.tokenID] = newAmount - oldAmount; - } - for (auto const & kv : balancesDiff) { - SetAccountHistory(kv.first, height, txn, txid, category, kv.second); - } - return true; -} diff --git a/src/masternodes/accountshistory.h b/src/masternodes/accountshistory.h index 6b5514b523..ff5b99467d 100644 --- a/src/masternodes/accountshistory.h +++ b/src/masternodes/accountshistory.h @@ -56,9 +56,8 @@ struct AccountHistoryValue { class CAccountsHistoryView : public virtual CStorageView { public: - Res SetAccountHistory(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diff); - void ForEachAccountHistory(std::function callback, AccountHistoryKey start) const; - bool TrackAffectedAccounts(CStorageKV const & before, MapKV const & diff, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category); + Res SetAccountHistory(AccountHistoryKey const & key, AccountHistoryValue const & value); + void ForEachAccountHistory(std::function callback, AccountHistoryKey start) const; // tags struct ByAccountHistoryKey { static const unsigned char prefix; }; diff --git a/src/masternodes/masternodes.cpp b/src/masternodes/masternodes.cpp index e557db4332..0bbf0c5b91 100644 --- a/src/masternodes/masternodes.cpp +++ b/src/masternodes/masternodes.cpp @@ -525,3 +525,36 @@ bool CCustomCSView::CanSpend(const uint256 & txId, int height) const return !pair || pair->second.destructionTx != uint256{} || pair->second.IsPoolShare(); } +CAccountsHistoryStorage::CAccountsHistoryStorage(CCustomCSView & storage, uint32_t height, uint32_t txn, const uint256& txid, uint8_t type) + : CStorageView(new CFlushableStorageKV(storage.GetRaw())), height(height), txn(txn), txid(txid), type(type) +{ + acindex = gArgs.GetBoolArg("-acindex", false); +} + +Res CAccountsHistoryStorage::AddBalance(CScript const & owner, CTokenAmount amount) +{ + auto res = CCustomCSView::AddBalance(owner, amount); + if (acindex && res.ok) { + diffs[owner][amount.nTokenId] += amount.nValue; + } + return res; +} + +Res CAccountsHistoryStorage::SubBalance(CScript const & owner, CTokenAmount amount) +{ + auto res = CCustomCSView::SubBalance(owner, amount); + if (acindex && res.ok) { + diffs[owner][amount.nTokenId] -= amount.nValue; + } + return res; +} + +bool CAccountsHistoryStorage::Flush() +{ + if (acindex) { + for (const auto& diff : diffs) { + SetAccountHistory({diff.first, height, txn}, {txid, type, diff.second}); + } + } + return CCustomCSView::Flush(); +} diff --git a/src/masternodes/masternodes.h b/src/masternodes/masternodes.h index 8ad1a3024f..c08f8e47f6 100644 --- a/src/masternodes/masternodes.h +++ b/src/masternodes/masternodes.h @@ -191,6 +191,8 @@ class CCustomCSView , public CGovView { public: + CCustomCSView() = default; + CCustomCSView(CStorageKV & st) : CStorageView(new CFlushableStorageKV(st)) {} @@ -217,6 +219,21 @@ class CCustomCSView } }; +class CAccountsHistoryStorage : public CCustomCSView +{ + bool acindex; + const uint32_t height; + const uint32_t txn; + const uint256 txid; + const uint8_t type; + std::map diffs; +public: + CAccountsHistoryStorage(CCustomCSView & storage, uint32_t height, uint32_t txn, const uint256& txid, uint8_t type); + Res AddBalance(CScript const & owner, CTokenAmount amount) override; + Res SubBalance(CScript const & owner, CTokenAmount amount) override; + bool Flush(); +}; + /** Global DB and view that holds enhanced chainstate data (should be protected by cs_main) */ extern std::unique_ptr pcustomcsDB; extern std::unique_ptr pcustomcsview; diff --git a/src/masternodes/mn_checks.cpp b/src/masternodes/mn_checks.cpp index 58b9dda8c2..bf9df6ab22 100644 --- a/src/masternodes/mn_checks.cpp +++ b/src/masternodes/mn_checks.cpp @@ -121,12 +121,20 @@ Res ApplyCustomTx(CCustomCSView & base_mnview, CCoinsViewCache const & coins, CT return Res::Ok(); // not "custom" tx } - CCustomCSView mnview(base_mnview); CustomTxType guess; + std::vector metadata; + try { - // Check if it is custom tx with metadata - std::vector metadata; guess = GuessCustomTxType(tx, metadata); + } catch (std::exception& e) { + return Res::Err("GuessCustomTxType: %s", e.what()); + } catch (...) { + return Res::Err("GuessCustomTxType: unexpected error"); + } + + CAccountsHistoryStorage mnview(base_mnview, height, txn, tx.GetHash(), (uint8_t)guess); + try { + // Check if it is custom tx with metadata switch (guess) { case CustomTxType::CreateMasternode: @@ -196,7 +204,6 @@ Res ApplyCustomTx(CCustomCSView & base_mnview, CCoinsViewCache const & coins, CT // construct undo auto& flushable = dynamic_cast(mnview.GetRaw()); - mnview.TrackAffectedAccounts(base_mnview.GetRaw(), flushable.GetRaw(), height, txn, tx.GetHash(), (unsigned char) guess); auto undo = CUndo::Construct(base_mnview.GetRaw(), flushable.GetRaw()); // flush changes mnview.Flush(); diff --git a/src/masternodes/mn_rpc.cpp b/src/masternodes/mn_rpc.cpp index 687b7cd0a0..f6ae021cc7 100644 --- a/src/masternodes/mn_rpc.cpp +++ b/src/masternodes/mn_rpc.cpp @@ -3000,15 +3000,15 @@ UniValue AmountsToJSON(TAmounts const & diffs) { return obj; } -UniValue accounthistoryToJSON(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diffs) { +UniValue accounthistoryToJSON(AccountHistoryKey const & key, AccountHistoryValue const & value) { UniValue obj(UniValue::VOBJ); - obj.pushKV("owner", ScriptToString(owner)); - obj.pushKV("blockHeight", (uint64_t) height); - obj.pushKV("type", ToString(CustomTxCodeToType(category))); - if (!txid.IsNull()) { - obj.pushKV("txn", (uint64_t) txn); - obj.pushKV("txid", txid.ToString()); + obj.pushKV("owner", ScriptToString(key.owner)); + obj.pushKV("blockHeight", (uint64_t) key.blockHeight); + obj.pushKV("type", ToString(CustomTxCodeToType(value.category))); + if (!value.txid.IsNull()) { + obj.pushKV("txn", (uint64_t) key.txn); + obj.pushKV("txid", value.txid.ToString()); } obj.pushKV("amounts", AmountsToJSON(diffs)); @@ -3136,7 +3136,7 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { bool isMine = false; filter = ISMINE_SPENDABLE; AccountHistoryKey startKey{ prevOwner, startBlock, std::numeric_limits::max() }; // starting from max txn values - pcustomcsview->ForEachAccountHistory([&](CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diffs) { + pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, AccountHistoryValue const & value) { if (height > startKey.blockHeight || depth <= startKey.blockHeight) return true; // continue @@ -3146,7 +3146,7 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { if(!tokenFilter.empty()) { bool hasToken = false; - for (auto const & diff : diffs) { + for (auto const & diff : value.diff) { auto token = pcustomcsview->GetToken(diff.first); std::string const tokenIdStr = token->CreateSymbolKey(diff.first); @@ -3162,18 +3162,18 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { } if(noRewards) { - if(category == static_cast(CustomTxType::NonTxRewards)) { + if(value.category == static_cast(CustomTxType::NonTxRewards)) { return true; // continue } } - if (prevOwner != owner) { - prevOwner = owner; - isMine = IsMine(*pwallet, owner) == ISMINE_SPENDABLE; + if (prevOwner != key.owner) { + prevOwner = key.owner; + isMine = IsMine(*pwallet, key.owner) == ISMINE_SPENDABLE; } if (isMine) { - ret.push_back(accounthistoryToJSON(owner, height, txn, txid, category, diffs)); + ret.push_back(accounthistoryToJSON(key, value)); --limit; if (shouldSearchInWallet) { @@ -3187,8 +3187,8 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { else if (accounts == "all") { // traversing the whole DB, skipping wrong heights AccountHistoryKey startKey{ CScript{}, startBlock, std::numeric_limits::max() }; // starting from max txn values - pcustomcsview->ForEachAccountHistory([&](CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diffs) { - if (height > startKey.blockHeight || depth <= startKey.blockHeight) { + pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, AccountHistoryValue const & value) { + if (key.blockHeight > startKey.blockHeight || (depth <= startKey.blockHeight && (key.blockHeight < startKey.blockHeight - depth))) { return true; // continue } @@ -3198,7 +3198,7 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { if(!tokenFilter.empty()) { bool hasToken = false; - for (auto const & diff : diffs) { + for (auto const & diff : value.diff) { auto token = pcustomcsview->GetToken(diff.first); std::string const tokenIdStr = token->CreateSymbolKey(diff.first); @@ -3214,12 +3214,12 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { } if(noRewards) { - if(category == static_cast(CustomTxType::NonTxRewards)) { + if(value.category == static_cast(CustomTxType::NonTxRewards)) { return true; // continue } } - ret.push_back(accounthistoryToJSON(owner, height, txn, txid, category, diffs)); + ret.push_back(accounthistoryToJSON(key, value)); if (shouldSearchInWallet) { txs.insert(txid); } @@ -3232,8 +3232,8 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { owner = DecodeScript(accounts); AccountHistoryKey startKey{ owner, startBlock, std::numeric_limits::max() }; // starting from max txn values - pcustomcsview->ForEachAccountHistory([&](CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diffs) { - if (owner != startKey.owner || height > startKey.blockHeight || depth <= startKey.blockHeight) + pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, AccountHistoryValue const & value) { + if (key.owner != startKey.owner || (key.blockHeight > startKey.blockHeight || (depth <= startKey.blockHeight && (key.blockHeight < startKey.blockHeight - depth)))) return false; if (CustomTxType::None != txType && category != uint8_t(txType)) { @@ -3242,7 +3242,7 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { if(!tokenFilter.empty()) { bool hasToken = false; - for (auto const & diff : diffs) { + for (auto const & diff : value.diff) { auto token = pcustomcsview->GetToken(diff.first); std::string const tokenIdStr = token->CreateSymbolKey(diff.first); @@ -3258,12 +3258,12 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { } if(noRewards) { - if(category == static_cast(CustomTxType::NonTxRewards)) { + if(value.category == static_cast(CustomTxType::NonTxRewards)) { return true; // continue } } - ret.push_back(accounthistoryToJSON(owner, height, txn, txid, category, diffs)); + ret.push_back(accounthistoryToJSON(key, value)); if (shouldSearchInWallet) { txs.insert(txid); } diff --git a/src/validation.cpp b/src/validation.cpp index 0bde469e51..17e6eca8bc 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2266,7 +2266,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl { // old data pruning and other (some processing made for the whole block) // make all changes to the new cache/snapshot to make it possible to take a diff later: - CCustomCSView cache(mnview); + CAccountsHistoryStorage cache(mnview, static_cast(pindex->nHeight), std::numeric_limits::max(), uint256(), uint8_t(CustomTxType::NonTxRewards)); // cache.CallYourInterblockProcessingsHere(); @@ -2303,7 +2303,6 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // construct undo auto& flushable = dynamic_cast(cache.GetRaw()); - cache.TrackAffectedAccounts(mnview.GetRaw(), flushable.GetRaw(), static_cast(pindex->nHeight), std::numeric_limits::max(), uint256(), (unsigned char) CustomTxType::NonTxRewards); auto undo = CUndo::Construct(mnview.GetRaw(), flushable.GetRaw()); // flush changes to underlying view cache.Flush(); From ecea413aebafd3b2ffa1eb513f3185a29581c8fb Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Mon, 7 Dec 2020 13:47:19 +0200 Subject: [PATCH 2/9] Lazy read db storage Signed-off-by: Anthony Fieroni --- src/flushablestorage.h | 25 +++++++++++++++++----- src/masternodes/accounts.cpp | 4 ++-- src/masternodes/accounts.h | 2 +- src/masternodes/accountshistory.cpp | 6 ++---- src/masternodes/accountshistory.h | 2 +- src/masternodes/criminals.cpp | 8 +++---- src/masternodes/govvariables/lp_splits.cpp | 6 +++--- src/masternodes/incentivefunding.cpp | 4 ++-- src/masternodes/incentivefunding.h | 2 +- src/masternodes/masternodes.cpp | 10 ++++----- src/masternodes/masternodes.h | 4 ++-- src/masternodes/mn_rpc.cpp | 8 +++---- src/masternodes/poolpairs.cpp | 10 ++++----- src/masternodes/poolpairs.h | 17 ++++++--------- src/masternodes/tokens.cpp | 12 +++++------ src/masternodes/tokens.h | 2 +- src/masternodes/undos.cpp | 6 ++---- src/masternodes/undos.h | 2 +- src/spv/spv_rpc.cpp | 2 +- src/test/liquidity_tests.cpp | 4 ++-- src/test/storage_tests.cpp | 6 +++--- 21 files changed, 74 insertions(+), 68 deletions(-) diff --git a/src/flushablestorage.h b/src/flushablestorage.h index a68b0a2e2f..52010be558 100644 --- a/src/flushablestorage.h +++ b/src/flushablestorage.h @@ -313,6 +313,24 @@ class CFlushableStorageKV : public CStorageKV { MapKV changed; }; +template +struct CLazySerialize +{ + CStorageKVIterator& it; + + operator T() + { + return get(); + } + + T get() + { + T value; + BytesToDbType(it.Value(), value); + return value; + } +}; + class CStorageView { public: CStorageView() = default; @@ -375,7 +393,7 @@ class CStorageView { } template - bool ForEach(std::function callback, KeyType const & start = KeyType()) const { + bool ForEach(std::function)> callback, KeyType const & start = KeyType()) const { auto& self = const_cast(*this); auto key = std::make_pair(By::prefix, start); @@ -383,10 +401,7 @@ class CStorageView { for(it->Seek(DbTypeToBytes(key)); it->Valid() && (BytesToDbType(it->Key(), key), key.first == By::prefix); it->Next()) { boost::this_thread::interruption_point(); - ValueType value; - BytesToDbType(it->Value(), value); - - if (!callback(key.second, value)) + if (!callback(key.second, CLazySerialize{*it})) break; } return true; diff --git a/src/masternodes/accounts.cpp b/src/masternodes/accounts.cpp index de3d35347c..9cca109716 100644 --- a/src/masternodes/accounts.cpp +++ b/src/masternodes/accounts.cpp @@ -7,9 +7,9 @@ /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!! const unsigned char CAccountsView::ByBalanceKey::prefix = 'a'; -void CAccountsView::ForEachBalance(std::function callback, BalanceKey start) const +void CAccountsView::ForEachBalance(std::function callback, BalanceKey const & start) const { - ForEach([&callback] (BalanceKey const & key, CAmount const & val) { + ForEach([&callback] (BalanceKey const & key, CAmount val) { return callback(key.owner, CTokenAmount{key.tokenID, val}); }, start); } diff --git a/src/masternodes/accounts.h b/src/masternodes/accounts.h index d966fd2ce6..318474615f 100644 --- a/src/masternodes/accounts.h +++ b/src/masternodes/accounts.h @@ -14,7 +14,7 @@ class CAccountsView : public virtual CStorageView { public: - void ForEachBalance(std::function callback, BalanceKey start = {}) const; + void ForEachBalance(std::function callback, BalanceKey const & start = {}) const; CTokenAmount GetBalance(CScript const & owner, DCT_ID tokenID) const; virtual Res AddBalance(CScript const & owner, CTokenAmount amount); diff --git a/src/masternodes/accountshistory.cpp b/src/masternodes/accountshistory.cpp index fb54c00f7c..2d80319cf9 100644 --- a/src/masternodes/accountshistory.cpp +++ b/src/masternodes/accountshistory.cpp @@ -11,11 +11,9 @@ /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!! const unsigned char CAccountsHistoryView::ByAccountHistoryKey::prefix = 'h'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB -void CAccountsHistoryView::ForEachAccountHistory(std::function callback, AccountHistoryKey start) const +void CAccountsHistoryView::ForEachAccountHistory(std::function)> callback, AccountHistoryKey const & start) const { - ForEach([&callback] (AccountHistoryKey const & key, AccountHistoryValue const & value) { - return callback(key, value); - }, start); + ForEach(callback, start); } Res CAccountsHistoryView::SetAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value) diff --git a/src/masternodes/accountshistory.h b/src/masternodes/accountshistory.h index ff5b99467d..75568f6e84 100644 --- a/src/masternodes/accountshistory.h +++ b/src/masternodes/accountshistory.h @@ -57,7 +57,7 @@ class CAccountsHistoryView : public virtual CStorageView { public: Res SetAccountHistory(AccountHistoryKey const & key, AccountHistoryValue const & value); - void ForEachAccountHistory(std::function callback, AccountHistoryKey start) const; + void ForEachAccountHistory(std::function)> callback, AccountHistoryKey const & start = {}) const; // tags struct ByAccountHistoryKey { static const unsigned char prefix; }; diff --git a/src/masternodes/criminals.cpp b/src/masternodes/criminals.cpp index b0c37635f0..a9bdf57fb9 100644 --- a/src/masternodes/criminals.cpp +++ b/src/masternodes/criminals.cpp @@ -43,11 +43,11 @@ bool CMintedHeadersView::FetchMintedHeaders(const uint256 & txid, const uint64_t } blockHeaders.clear(); - ForEach([&txid, &mintedBlocks, &blockHeaders] (DBMNBlockHeadersKey const & key, CBlockHeader & blockHeader) { + ForEach([&txid, &mintedBlocks, &blockHeaders] (DBMNBlockHeadersKey const & key, CLazySerialize blockHeader) { if (key.masternodeID == txid && key.mintedBlocks == mintedBlocks) { - blockHeaders.emplace(key.blockHash, std::move(blockHeader)); + blockHeaders.emplace(key.blockHash, blockHeader.get()); return true; // continue } return false; // break! @@ -76,12 +76,12 @@ void CCriminalProofsView::RemoveCriminalProofs(const uint256 & mnId) { CCriminalProofsView::CMnCriminals CCriminalProofsView::GetUnpunishedCriminals() { CMnCriminals result; - ForEach([&result] (uint256 const & id, CDoubleSignFact & proof) { + ForEach([&result] (uint256 const & id, CLazySerialize proof) { // matching with already punished. and this is the ONLY measure! auto node = pcustomcsview->GetMasternode(id); // assert? if (node && node->banTx.IsNull()) { - result.emplace(id, std::move(proof)); + result.emplace(id, proof.get()); } return true; // continue }); diff --git a/src/masternodes/govvariables/lp_splits.cpp b/src/masternodes/govvariables/lp_splits.cpp index f01db1e514..1f66ed8c19 100644 --- a/src/masternodes/govvariables/lp_splits.cpp +++ b/src/masternodes/govvariables/lp_splits.cpp @@ -50,12 +50,12 @@ Res LP_SPLITS::Validate(const CCustomCSView & mnview) const { } Res LP_SPLITS::Apply(CCustomCSView & mnview) { - mnview.ForEachPoolPair([&] (const DCT_ID poolId, const CPoolPair & pool) { + mnview.ForEachPoolPair([&] (const DCT_ID poolId, CPoolPair pool) { // we ought to reset previous value: - const_cast(pool).rewardPct = 0; + pool.rewardPct = 0; auto it = splits.find(poolId); if (it != splits.end()) { - const_cast(pool).rewardPct = it->second; + pool.rewardPct = it->second; } mnview.SetPoolPair(poolId, pool); diff --git a/src/masternodes/incentivefunding.cpp b/src/masternodes/incentivefunding.cpp index a23ab365f3..0f3cdd6693 100644 --- a/src/masternodes/incentivefunding.cpp +++ b/src/masternodes/incentivefunding.cpp @@ -28,9 +28,9 @@ Res CCommunityBalancesView::SetCommunityBalance(CommunityAccountType account, CA return Res::Ok(); } -void CCommunityBalancesView::ForEachCommunityBalance(std::function callback) const +void CCommunityBalancesView::ForEachCommunityBalance(std::function)> callback) const { - ForEach([&callback] (unsigned char const & key, CAmount const & val) { + ForEach([&callback] (unsigned char const & key, CLazySerialize val) { return callback(CommunityAccountCodeToType(key), val); }, '\0'); diff --git a/src/masternodes/incentivefunding.h b/src/masternodes/incentivefunding.h index 94eb7cee41..00fbdc272c 100644 --- a/src/masternodes/incentivefunding.h +++ b/src/masternodes/incentivefunding.h @@ -35,7 +35,7 @@ class CCommunityBalancesView : public virtual CStorageView CAmount GetCommunityBalance(CommunityAccountType account) const; Res SetCommunityBalance(CommunityAccountType account, CAmount amount); - void ForEachCommunityBalance(std::function callback) const; + void ForEachCommunityBalance(std::function)> callback) const; Res AddCommunityBalance(CommunityAccountType account, CAmount amount); Res SubCommunityBalance(CommunityAccountType account, CAmount amount); diff --git a/src/masternodes/masternodes.cpp b/src/masternodes/masternodes.cpp index 0bbf0c5b91..b6187cd32b 100644 --- a/src/masternodes/masternodes.cpp +++ b/src/masternodes/masternodes.cpp @@ -209,11 +209,9 @@ boost::optional CMasternodesView::GetMasternodeIdByOwner(const CKeyID & return ReadBy(id); } -void CMasternodesView::ForEachMasternode(std::function callback, uint256 const & start) +void CMasternodesView::ForEachMasternode(std::function)> callback, uint256 const & start) { - ForEach([&callback] (uint256 const & txid, CMasternode & node) { - return callback(txid, node); - }, start); + ForEach(callback, start); } void CMasternodesView::IncrementMintedBy(const CKeyID & minter) @@ -426,7 +424,7 @@ void CAnchorRewardsView::RemoveRewardForAnchor(const CAnchorRewardsView::AnchorT EraseBy(btcTxHash); } -void CAnchorRewardsView::ForEachAnchorReward(std::function callback) +void CAnchorRewardsView::ForEachAnchorReward(std::function)> callback) { ForEach(callback); } @@ -442,7 +440,7 @@ CTeamView::CTeam CCustomCSView::CalcNextTeam(const uint256 & stakeModifier) int anchoringTeamSize = Params().GetConsensus().mn.anchoringTeamSize; std::map> priorityMN; - ForEachMasternode([&stakeModifier, &priorityMN] (uint256 const & id, CMasternode & node) { + ForEachMasternode([&stakeModifier, &priorityMN] (uint256 const & id, CMasternode node) { if(!node.IsActive()) return true; diff --git a/src/masternodes/masternodes.h b/src/masternodes/masternodes.h index c08f8e47f6..1c9475064a 100644 --- a/src/masternodes/masternodes.h +++ b/src/masternodes/masternodes.h @@ -117,7 +117,7 @@ class CMasternodesView : public virtual CStorageView boost::optional GetMasternode(uint256 const & id) const; boost::optional GetMasternodeIdByOperator(CKeyID const & id) const; boost::optional GetMasternodeIdByOwner(CKeyID const & id) const; - void ForEachMasternode(std::function callback, uint256 const & start = uint256()); + void ForEachMasternode(std::function)> callback, uint256 const & start = uint256()); void IncrementMintedBy(CKeyID const & minter); void DecrementMintedBy(CKeyID const & minter); @@ -171,7 +171,7 @@ class CAnchorRewardsView : public virtual CStorageView void AddRewardForAnchor(AnchorTxHash const &btcTxHash, RewardTxHash const & rewardTxHash); void RemoveRewardForAnchor(AnchorTxHash const &btcTxHash); - void ForEachAnchorReward(std::function callback); + void ForEachAnchorReward(std::function)> callback); struct BtcTx { static const unsigned char prefix; }; }; diff --git a/src/masternodes/mn_rpc.cpp b/src/masternodes/mn_rpc.cpp index f6ae021cc7..ef15d2a68c 100644 --- a/src/masternodes/mn_rpc.cpp +++ b/src/masternodes/mn_rpc.cpp @@ -721,7 +721,7 @@ UniValue listmasternodes(const JSONRPCRequest& request) { UniValue ret(UniValue::VOBJ); LOCK(cs_main); - pcustomcsview->ForEachMasternode([&](uint256 const& nodeId, CMasternode& node) { + pcustomcsview->ForEachMasternode([&](uint256 const& nodeId, CMasternode node) { ret.pushKVs(mnToJSON(nodeId, node, verbose)); limit--; return limit != 0; @@ -1239,7 +1239,7 @@ UniValue listtokens(const JSONRPCRequest& request) { LOCK(cs_main); UniValue ret(UniValue::VOBJ); - pcustomcsview->ForEachToken([&](DCT_ID const& id, CTokenImplementation const& token) { + pcustomcsview->ForEachToken([&](DCT_ID const& id, CTokenImplementation token) { ret.pushKVs(tokenToJSON(id, token, verbose)); limit--; @@ -1820,10 +1820,10 @@ UniValue listpoolpairs(const JSONRPCRequest& request) { LOCK(cs_main); UniValue ret(UniValue::VOBJ); - pcustomcsview->ForEachPoolPair([&](DCT_ID const & id, CPoolPair const & pool) { + pcustomcsview->ForEachPoolPair([&](DCT_ID const & id, CLazySerialize pool) { const auto token = pcustomcsview->GetToken(id); if (token) { - ret.pushKVs(poolToJSON(id, pool, *token, verbose)); + ret.pushKVs(poolToJSON(id, pool.get(), *token, verbose)); } limit--; diff --git a/src/masternodes/poolpairs.cpp b/src/masternodes/poolpairs.cpp index 5700fee960..2f20a46d0c 100644 --- a/src/masternodes/poolpairs.cpp +++ b/src/masternodes/poolpairs.cpp @@ -156,7 +156,7 @@ CAmount CPoolPair::slopeSwap(CAmount unswapped, CAmount &poolFrom, CAmount &pool arith_uint256 poolF = arith_uint256(poolFrom); arith_uint256 poolT = arith_uint256(poolTo); - + arith_uint256 swapped = 0; if (!postBayfrontGardens) { CAmount chunk = poolFrom/SLOPE_SWAP_RATE < unswapped ? poolFrom/SLOPE_SWAP_RATE : unswapped; @@ -183,18 +183,18 @@ CAmount CPoolPair::slopeSwap(CAmount unswapped, CAmount &poolFrom, CAmount &pool return swapped.GetLow64(); } -void CPoolPairView::ForEachPoolPair(std::function callback, DCT_ID const & start) { +void CPoolPairView::ForEachPoolPair(std::function)> callback, DCT_ID const & start) { DCT_ID poolId = start; auto hint = WrapVarInt(poolId.v); - ForEach, CPoolPair>([&poolId, &callback] (CVarInt const &, CPoolPair & pool) { + ForEach, CPoolPair>([&poolId, &callback] (CVarInt const &, CLazySerialize pool) { return callback(poolId, pool); }, hint); } -void CPoolPairView::ForEachPoolShare(std::function callback, const PoolShareKey &startKey) const +void CPoolPairView::ForEachPoolShare(std::function callback, const PoolShareKey &startKey) const { - ForEach([&callback] (PoolShareKey const & poolShareKey, const char &) { + ForEach([&callback] (PoolShareKey const & poolShareKey, CLazySerialize) { return callback(poolShareKey.poolID, poolShareKey.owner); }, startKey); } diff --git a/src/masternodes/poolpairs.h b/src/masternodes/poolpairs.h index bd5a7412a5..f5e9062963 100644 --- a/src/masternodes/poolpairs.h +++ b/src/masternodes/poolpairs.h @@ -230,7 +230,6 @@ struct PoolShareKey { } }; - class CPoolPairView : public virtual CStorageView { public: @@ -241,11 +240,11 @@ class CPoolPairView : public virtual CStorageView boost::optional GetPoolPair(const DCT_ID &poolId) const; boost::optional > GetPoolPair(DCT_ID const & tokenA, DCT_ID const & tokenB) const; - void ForEachPoolPair(std::function callback, DCT_ID const & start = DCT_ID{0}); - void ForEachPoolShare(std::function callback, PoolShareKey const &startKey = PoolShareKey{0,CScript{}}) const; + void ForEachPoolPair(std::function)> callback, DCT_ID const & start = DCT_ID{0}); + void ForEachPoolShare(std::function callback, PoolShareKey const &startKey = PoolShareKey{0,CScript{}}) const; Res SetShare(DCT_ID const & poolId, CScript const & provider) { - WriteBy(PoolShareKey{ poolId, provider}, '\0'); + WriteBy(PoolShareKey{poolId, provider}, '\0'); return Res::Ok(); } Res DelShare(DCT_ID const & poolId, CScript const & provider) { @@ -259,14 +258,13 @@ class CPoolPairView : public virtual CStorageView uint32_t const PRECISION = 10000; // (== 100%) just searching the way to avoid arith256 inflating CAmount totalDistributed = 0; - ForEachPoolPair([&] (DCT_ID const & poolId, CPoolPair const & pool) { + ForEachPoolPair([&] (DCT_ID const & poolId, CPoolPair pool) { // yield farming counters CAmount const poolReward = yieldFarming * pool.rewardPct / COIN; // 'rewardPct' should be defined by 'setgov "LP_SPLITS"', also, it is assumed that it was totally validated and normalized to 100% CAmount distributedFeeA = 0; CAmount distributedFeeB = 0; - if (pool.totalLiquidity == 0 || (!pool.swapEvent && poolReward == 0)) { return true; // no events, skip to the next pool } @@ -310,10 +308,9 @@ class CPoolPairView : public virtual CStorageView return true; }, PoolShareKey{poolId, CScript{}}); - // we have no "non-const foreaches", but it is safe here cause not broke indexes, so: - const_cast(pool).blockCommissionA -= distributedFeeA; - const_cast(pool).blockCommissionB -= distributedFeeB; - const_cast(pool).swapEvent = false; + pool.blockCommissionA -= distributedFeeA; + pool.blockCommissionB -= distributedFeeB; + pool.swapEvent = false; auto res = SetPoolPair(poolId, pool); if (!res.ok) diff --git a/src/masternodes/tokens.cpp b/src/masternodes/tokens.cpp index cd81959055..fbea127783 100644 --- a/src/masternodes/tokens.cpp +++ b/src/masternodes/tokens.cpp @@ -90,12 +90,12 @@ std::unique_ptr CTokensView::GetTokenGuessId(const std::string & str, DC return {}; } -void CTokensView::ForEachToken(std::function callback, DCT_ID const & start) +void CTokensView::ForEachToken(std::function)> callback, DCT_ID const & start) { DCT_ID tokenId = start; auto hint = WrapVarInt(tokenId.v); - ForEach, CTokenImpl>([&tokenId, &callback] (CVarInt const &, CTokenImpl & tokenImpl) { + ForEach, CTokenImpl>([&tokenId, &callback] (CVarInt const &, CLazySerialize tokenImpl) { return callback(tokenId, tokenImpl); }, hint); @@ -137,7 +137,7 @@ ResVal CTokensView::CreateToken(const CTokensView::CTokenImpl & token, b if (GetToken(token.symbol)) { return Res::Err("token '%s' already exists!", token.symbol); } - ForEachToken([&](DCT_ID const& currentId, CTokenImplementation const& ) { + ForEachToken([&](DCT_ID const& currentId, CLazySerialize) { if(currentId < DCT_ID_START) id.v = currentId.v + 1; return currentId < DCT_ID_START; @@ -241,15 +241,15 @@ Res CTokensView::UpdateToken(const uint256 &tokenTx, CToken & newToken, bool isP */ Res CTokensView::BayfrontFlagsCleanup() { - ForEachToken([&] (DCT_ID const & id, CTokenImpl const & token){ + ForEachToken([&] (DCT_ID const & id, CTokenImpl token){ bool changed{false}; if (token.IsFinalized()) { - const_cast(token).flags ^= (uint8_t)CToken::TokenFlags::Finalized; + token.flags ^= (uint8_t)CToken::TokenFlags::Finalized; LogPrintf("Warning! Got `Finalized` token, id=%s\n", id.ToString().c_str()); changed = true; } if (token.IsPoolShare()) { - const_cast(token).flags ^= (uint8_t)CToken::TokenFlags::LPS; + token.flags ^= (uint8_t)CToken::TokenFlags::LPS; LogPrintf("Warning! Got `LPS` token, id=%s\n", id.ToString().c_str()); changed = true; } diff --git a/src/masternodes/tokens.h b/src/masternodes/tokens.h index e4e369dda9..604a7f85de 100644 --- a/src/masternodes/tokens.h +++ b/src/masternodes/tokens.h @@ -141,7 +141,7 @@ class CTokensView : public virtual CStorageView boost::optional> GetTokenByCreationTx(uint256 const & txid) const; std::unique_ptr GetTokenGuessId(const std::string & str, DCT_ID & id) const; - void ForEachToken(std::function callback, DCT_ID const & start = DCT_ID{0}); + void ForEachToken(std::function)> callback, DCT_ID const & start = DCT_ID{0}); Res CreateDFIToken(); ResVal CreateToken(CTokenImpl const & token, bool isPreBayfront); diff --git a/src/masternodes/undos.cpp b/src/masternodes/undos.cpp index 2b6e131e34..e962cadc9e 100644 --- a/src/masternodes/undos.cpp +++ b/src/masternodes/undos.cpp @@ -7,11 +7,9 @@ /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!! const unsigned char CUndosView::ByUndoKey::prefix = 'u'; -void CUndosView::ForEachUndo(std::function callback, UndoKey start) const +void CUndosView::ForEachUndo(std::function)> callback, UndoKey const & start) const { - ForEach([&callback] (UndoKey const & key, CUndo const & val) { - return callback(key, val); - }, start); + ForEach(callback, start); } Res CUndosView::SetUndo(UndoKey key, CUndo const & undo) diff --git a/src/masternodes/undos.h b/src/masternodes/undos.h index 1d33a1076c..3917a78dbe 100644 --- a/src/masternodes/undos.h +++ b/src/masternodes/undos.h @@ -11,7 +11,7 @@ class CUndosView : public virtual CStorageView { public: - void ForEachUndo(std::function callback, UndoKey start) const; + void ForEachUndo(std::function)> callback, UndoKey const & start = {}) const; boost::optional GetUndo(UndoKey key) const; Res SetUndo(UndoKey key, CUndo const & undo); diff --git a/src/spv/spv_rpc.cpp b/src/spv/spv_rpc.cpp index 0639151f38..12c43c3ce7 100644 --- a/src/spv/spv_rpc.cpp +++ b/src/spv/spv_rpc.cpp @@ -674,7 +674,7 @@ UniValue spv_listanchorrewards(const JSONRPCRequest& request) UniValue result(UniValue::VARR); - pcustomcsview->ForEachAnchorReward([&result] (uint256 const & btcHash, uint256 & rewardHash) { + pcustomcsview->ForEachAnchorReward([&result] (uint256 const & btcHash, uint256 rewardHash) { UniValue item(UniValue::VOBJ); item.pushKV("AnchorTxHash", btcHash.ToString()); item.pushKV("RewardTxHash", rewardHash.ToString()); diff --git a/src/test/liquidity_tests.cpp b/src/test/liquidity_tests.cpp index 57a1ce343f..a8e5b84b23 100644 --- a/src/test/liquidity_tests.cpp +++ b/src/test/liquidity_tests.cpp @@ -273,7 +273,7 @@ BOOST_AUTO_TEST_CASE(math_rewards) } // create shares - mnview.ForEachPoolPair([&] (DCT_ID const & idPool, CPoolPair const & pool) { + mnview.ForEachPoolPair([&] (DCT_ID const & idPool, CLazySerialize) { // printf("pool id = %s\n", idPool.ToString().c_str()); for (int i = 0; i < ProvidersCount; ++i) { CScript shareAddress = CScript(idPool.v * ProvidersCount + i); @@ -302,7 +302,7 @@ BOOST_AUTO_TEST_CASE(math_rewards) /// DCT_ID{10} - 0 // set "traded fees" here too, just to estimate proc.load - cache.ForEachPoolPair([&] (DCT_ID const & idPool, CPoolPair const & pool) { + cache.ForEachPoolPair([&] (DCT_ID const & idPool, CLazySerialize) { SetPoolTradeFees(cache, idPool, idPool.v * COIN, idPool.v * COIN*2); return true; }); diff --git a/src/test/storage_tests.cpp b/src/test/storage_tests.cpp index 15f7d67322..d0aaa267c9 100644 --- a/src/test/storage_tests.cpp +++ b/src/test/storage_tests.cpp @@ -49,7 +49,7 @@ UniValue CallRPC(std::string args) int GetTokensCount() { int counter{0}; - pcustomcsview->ForEachToken([&counter] (DCT_ID const & id, CTokenImplementation const & token) { + pcustomcsview->ForEachToken([&counter] (DCT_ID const & id, CLazySerialize) { // printf("DCT_ID: %d, Token: %s: %s\n", id, token.symbol.c_str(), token.name.c_str()); // dump for debug ++counter; return true; @@ -307,7 +307,7 @@ BOOST_AUTO_TEST_CASE(for_each_order) pcustomcsview->WriteBy(TestForward{((uint32_t)-1)}, 8); int test = 1; - pcustomcsview->ForEach([&] (TestForward const & key, int & value) { + pcustomcsview->ForEach([&] (TestForward const & key, int value) { // printf("%ld : %d\n", key.n, value); BOOST_CHECK(value == test); ++test; @@ -323,7 +323,7 @@ BOOST_AUTO_TEST_CASE(for_each_order) pcustomcsview->WriteBy(TestBackward{(uint16_t)-1}, 6); int test = 6; - pcustomcsview->ForEach([&] (TestBackward const & key, int & value) { + pcustomcsview->ForEach([&] (TestBackward const & key, int value) { // printf("%ld : %d\n", key.n, value); BOOST_CHECK(value == test); --test; From 65a6b2408b93cc88cf3877bed1e54d9b0ab80d1a Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Mon, 7 Dec 2020 15:56:21 +0200 Subject: [PATCH 3/9] Keep reward history separate Signed-off-by: Anthony Fieroni --- src/init.cpp | 15 ++ src/masternodes/accountshistory.cpp | 12 ++ src/masternodes/accountshistory.h | 47 ++++++ src/masternodes/masternodes.cpp | 29 ++++ src/masternodes/masternodes.h | 13 ++ src/masternodes/mn_checks.h | 6 +- src/masternodes/mn_rpc.cpp | 234 +++++++++++----------------- src/masternodes/poolpairs.h | 23 ++- src/test/liquidity_tests.cpp | 2 +- src/validation.cpp | 6 +- 10 files changed, 231 insertions(+), 156 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index cdc9fc269b..ac97cea6d6 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1585,6 +1585,21 @@ bool AppInitMain(InitInterfaces& interfaces) pcustomcsDB = MakeUnique(GetDataDir() / "enhancedcs", nMinDbCache << 20, false, fReset || fReindexChainState); pcustomcsview.reset(); pcustomcsview = MakeUnique(*pcustomcsDB.get()); + if (!fReset && gArgs.GetBoolArg("-acindex", false)) { + bool hasAccountHistory = false, hasRewardHistory = false; + pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const &, CLazySerialize) { + hasAccountHistory = true; + return false; + }); + pcustomcsview->ForEachRewardHistory([&](RewardHistoryKey const &, CLazySerialize) { + hasRewardHistory = true; + return false; + }); + if (hasAccountHistory && !hasRewardHistory) { + strLoadError = _("Account history needs rebuild").translated; + break; + } + } panchorauths.reset(); panchorauths = MakeUnique(); diff --git a/src/masternodes/accountshistory.cpp b/src/masternodes/accountshistory.cpp index 2d80319cf9..640122c12a 100644 --- a/src/masternodes/accountshistory.cpp +++ b/src/masternodes/accountshistory.cpp @@ -10,6 +10,7 @@ /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!! const unsigned char CAccountsHistoryView::ByAccountHistoryKey::prefix = 'h'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB +const unsigned char CRewardsHistoryView::ByRewardHistoryKey::prefix = 'R'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB void CAccountsHistoryView::ForEachAccountHistory(std::function)> callback, AccountHistoryKey const & start) const { @@ -21,3 +22,14 @@ Res CAccountsHistoryView::SetAccountHistory(const AccountHistoryKey& key, const WriteBy(key, value); return Res::Ok(); } + +void CRewardsHistoryView::ForEachRewardHistory(std::function)> callback, RewardHistoryKey const & start) const +{ + ForEach(callback, start); +} + +Res CRewardsHistoryView::SetRewardHistory(const RewardHistoryKey& key, const RewardHistoryValue& value) +{ + WriteBy(key, value); + return Res::Ok(); +} diff --git a/src/masternodes/accountshistory.h b/src/masternodes/accountshistory.h index 75568f6e84..aa3879bed4 100644 --- a/src/masternodes/accountshistory.h +++ b/src/masternodes/accountshistory.h @@ -53,6 +53,43 @@ struct AccountHistoryValue { } }; +struct RewardHistoryKey { + CScript owner; + uint32_t blockHeight; + DCT_ID poolID; // for order in block + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(owner); + + if (ser_action.ForRead()) { + READWRITE(WrapBigEndian(blockHeight)); + blockHeight = ~blockHeight; + } + else { + uint32_t blockHeight_ = ~blockHeight; + READWRITE(WrapBigEndian(blockHeight_)); + } + + READWRITE(VARINT(poolID.v)); + } +}; + +struct RewardHistoryValue { + unsigned char category; + TAmounts diff; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(category); + READWRITE(diff); + } +}; + class CAccountsHistoryView : public virtual CStorageView { public: @@ -63,4 +100,14 @@ class CAccountsHistoryView : public virtual CStorageView struct ByAccountHistoryKey { static const unsigned char prefix; }; }; +class CRewardsHistoryView : public virtual CStorageView +{ +public: + Res SetRewardHistory(RewardHistoryKey const & key, RewardHistoryValue const & value); + void ForEachRewardHistory(std::function)> callback, RewardHistoryKey const & start = {}) const; + + // tags + struct ByRewardHistoryKey { static const unsigned char prefix; }; +}; + #endif //DEFI_MASTERNODES_ACCOUNTSHISTORY_H diff --git a/src/masternodes/masternodes.cpp b/src/masternodes/masternodes.cpp index b6187cd32b..936522b02e 100644 --- a/src/masternodes/masternodes.cpp +++ b/src/masternodes/masternodes.cpp @@ -556,3 +556,32 @@ bool CAccountsHistoryStorage::Flush() } return CCustomCSView::Flush(); } + +CRewardsHistoryStorage::CRewardsHistoryStorage(CCustomCSView & storage, uint32_t height) + : CStorageView(new CFlushableStorageKV(storage.GetRaw())), height(height) +{ + acindex = gArgs.GetBoolArg("-acindex", false); +} + +Res CRewardsHistoryStorage::AddBalance(CScript const & owner, DCT_ID poolID, uint8_t type, CTokenAmount amount) +{ + auto res = CCustomCSView::AddBalance(owner, amount); + if (acindex && res.ok) { + auto& tuple = diffs[owner]; + std::get<0>(tuple) = poolID; + std::get<1>(tuple) = type; + std::get<2>(tuple)[amount.nTokenId] += amount.nValue; + } + return res; +} + +bool CRewardsHistoryStorage::Flush() +{ + if (acindex) { + for (const auto& diff : diffs) { + const auto& tuple = diff.second; + SetRewardHistory({diff.first, height, std::get<0>(tuple)}, {std::get<1>(tuple), std::get<2>(tuple)}); + } + } + return CCustomCSView::Flush(); +} diff --git a/src/masternodes/masternodes.h b/src/masternodes/masternodes.h index 1c9475064a..fc16d0772a 100644 --- a/src/masternodes/masternodes.h +++ b/src/masternodes/masternodes.h @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -185,6 +186,7 @@ class CCustomCSView , public CTokensView , public CAccountsView , public CAccountsHistoryView + , public CRewardsHistoryView , public CCommunityBalancesView , public CUndosView , public CPoolPairView @@ -234,6 +236,17 @@ class CAccountsHistoryStorage : public CCustomCSView bool Flush(); }; +class CRewardsHistoryStorage : public CCustomCSView +{ + bool acindex; + const uint32_t height; + std::map> diffs; +public: + CRewardsHistoryStorage(CCustomCSView & storage, uint32_t height); + Res AddBalance(CScript const & owner, DCT_ID poolID, uint8_t type, CTokenAmount amount); + bool Flush(); +}; + /** Global DB and view that holds enhanced chainstate data (should be protected by cs_main) */ extern std::unique_ptr pcustomcsDB; extern std::unique_ptr pcustomcsview; diff --git a/src/masternodes/mn_checks.h b/src/masternodes/mn_checks.h index 8c1057249b..4b579bab25 100644 --- a/src/masternodes/mn_checks.h +++ b/src/masternodes/mn_checks.h @@ -54,13 +54,10 @@ enum class CustomTxType : unsigned char AnyAccountsToAccounts = 'a', //set governance variable SetGovVariable = 'G', - - // this is not the real tx type (!) but special category for accounts/history tracking - NonTxRewards = '+' }; inline CustomTxType CustomTxCodeToType(unsigned char ch) { - char const txtypes[] = "CRTMNnpuslrUbBaG+"; + char const txtypes[] = "CRTMNnpuslrUbBaG"; if (memchr(txtypes, ch, strlen(txtypes))) return static_cast(ch); else @@ -86,7 +83,6 @@ inline std::string ToString(CustomTxType type) { case CustomTxType::AccountToAccount: return "AccountToAccount"; case CustomTxType::AnyAccountsToAccounts: return "AnyAccountsToAccounts"; case CustomTxType::SetGovVariable: return "SetGovVariable"; - case CustomTxType::NonTxRewards: return "Rewards"; default: return "None"; } } diff --git a/src/masternodes/mn_rpc.cpp b/src/masternodes/mn_rpc.cpp index ef15d2a68c..bb2972b6e2 100644 --- a/src/masternodes/mn_rpc.cpp +++ b/src/masternodes/mn_rpc.cpp @@ -3006,21 +3006,27 @@ UniValue accounthistoryToJSON(AccountHistoryKey const & key, AccountHistoryValue obj.pushKV("owner", ScriptToString(key.owner)); obj.pushKV("blockHeight", (uint64_t) key.blockHeight); obj.pushKV("type", ToString(CustomTxCodeToType(value.category))); - if (!value.txid.IsNull()) { - obj.pushKV("txn", (uint64_t) key.txn); - obj.pushKV("txid", value.txid.ToString()); - } + obj.pushKV("txn", (uint64_t) key.txn); + obj.pushKV("txid", value.txid.ToString()); + obj.pushKV("amounts", AmountsToJSON(value.diff)); + return obj; +} - obj.pushKV("amounts", AmountsToJSON(diffs)); +UniValue rewardhistoryToJSON(RewardHistoryKey const & key, RewardHistoryValue const & value) { + UniValue obj(UniValue::VOBJ); + obj.pushKV("owner", ScriptToString(key.owner)); + obj.pushKV("blockHeight", (uint64_t) key.blockHeight); + obj.pushKV("type", RewardToString(RewardType(value.category))); + obj.pushKV("poolID", key.poolID.ToString()); + obj.pushKV("amounts", AmountsToJSON(value.diff)); return obj; } -UniValue outputEntryToJSON(COutputEntry const & entry, uint32_t height, uint256 const & hashBlock, uint256 const & txid, std::string const & type) { +UniValue outputEntryToJSON(COutputEntry const & entry, uint32_t height, uint256 const & txid, std::string const & type) { UniValue obj(UniValue::VOBJ); obj.pushKV("owner", EncodeDestination(entry.destination)); obj.pushKV("blockHeight", (uint64_t)height); - obj.pushKV("blockHash", hashBlock.GetHex()); obj.pushKV("type", type); obj.pushKV("txn", (uint64_t) entry.vout); obj.pushKV("txid", txid.ToString()); @@ -3126,149 +3132,67 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { std::set txs; const bool shouldSearchInWallet = tokenFilter.empty() || tokenFilter == "DFI"; - UniValue ret(UniValue::VARR); - - LOCK2(cs_main, pwallet->cs_wallet); - - if (accounts == "mine") { - // traversing through owned scripts - CScript prevOwner{}; - bool isMine = false; - filter = ISMINE_SPENDABLE; - AccountHistoryKey startKey{ prevOwner, startBlock, std::numeric_limits::max() }; // starting from max txn values - pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, AccountHistoryValue const & value) { - if (height > startKey.blockHeight || depth <= startKey.blockHeight) - return true; // continue - - if (CustomTxType::None != txType && category != uint8_t(txType)) { - return true; - } - - if(!tokenFilter.empty()) { - bool hasToken = false; - for (auto const & diff : value.diff) { - auto token = pcustomcsview->GetToken(diff.first); - std::string const tokenIdStr = token->CreateSymbolKey(diff.first); - - if(tokenIdStr == tokenFilter) { - hasToken = true; - break; - } - } - - if(!hasToken) { - return true; // continue - } - } - - if(noRewards) { - if(value.category == static_cast(CustomTxType::NonTxRewards)) { - return true; // continue - } - } - - if (prevOwner != key.owner) { - prevOwner = key.owner; - isMine = IsMine(*pwallet, key.owner) == ISMINE_SPENDABLE; - } - - if (isMine) { - ret.push_back(accounthistoryToJSON(key, value)); - --limit; + CScript account; + std::function preCondition; + std::function isForMe = [](CScript const&) { return true; }; - if (shouldSearchInWallet) { - txs.insert(txid); - } - } - - return limit != 0; - }, startKey); + if (accounts == "mine" || accounts == "all") { + preCondition = [startBlock, depth](uint32_t blockHeight, CScript const&) { + return blockHeight > startBlock || depth <= startBlock; + }; + if (accounts == "mine") { + isForMe = [pwallet](CScript const & owner) { + return IsMine(*pwallet, owner) == ISMINE_SPENDABLE; + }; + } + } else { + account = DecodeScript(accounts); + preCondition = [startBlock, &account, depth](uint32_t blockHeight, CScript const & owner) { + return owner != account || blockHeight > startBlock || depth <= startBlock; + }; } - else if (accounts == "all") { - // traversing the whole DB, skipping wrong heights - AccountHistoryKey startKey{ CScript{}, startBlock, std::numeric_limits::max() }; // starting from max txn values - pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, AccountHistoryValue const & value) { - if (key.blockHeight > startKey.blockHeight || (depth <= startKey.blockHeight && (key.blockHeight < startKey.blockHeight - depth))) { - return true; // continue - } - if (CustomTxType::None != txType && category != uint8_t(txType)) { + auto hasToken = [&tokenFilter](TAmounts const & diffs) { + for (auto const & diff : diffs) { + auto token = pcustomcsview->GetToken(diff.first); + auto const tokenIdStr = token->CreateSymbolKey(diff.first); + if(tokenIdStr == tokenFilter) { return true; } + } + return false; + }; - if(!tokenFilter.empty()) { - bool hasToken = false; - for (auto const & diff : value.diff) { - auto token = pcustomcsview->GetToken(diff.first); - std::string const tokenIdStr = token->CreateSymbolKey(diff.first); + LOCK(cs_main); + UniValue ret(UniValue::VARR); - if(tokenIdStr == tokenFilter) { - hasToken = true; - break; - } - } + pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, CLazySerialize valueLazy) { + if (preCondition(key.blockHeight, key.owner)) + return true; // continue - if(!hasToken) { - return true; // continue - } - } + auto value = valueLazy.get(); - if(noRewards) { - if(value.category == static_cast(CustomTxType::NonTxRewards)) { - return true; // continue - } - } + if(!tokenFilter.empty() && !hasToken(value.diff)) { + return true; + } + if (CustomTxType::None != txType && value.category != uint8_t(txType)) { + return true; + } + + if (isForMe(key.owner)) { ret.push_back(accounthistoryToJSON(key, value)); if (shouldSearchInWallet) { - txs.insert(txid); - } - return --limit != 0; - }, startKey); - - } - else { - // parse single script/address: - owner = DecodeScript(accounts); - - AccountHistoryKey startKey{ owner, startBlock, std::numeric_limits::max() }; // starting from max txn values - pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, AccountHistoryValue const & value) { - if (key.owner != startKey.owner || (key.blockHeight > startKey.blockHeight || (depth <= startKey.blockHeight && (key.blockHeight < startKey.blockHeight - depth)))) - return false; - - if (CustomTxType::None != txType && category != uint8_t(txType)) { - return true; - } - - if(!tokenFilter.empty()) { - bool hasToken = false; - for (auto const & diff : value.diff) { - auto token = pcustomcsview->GetToken(diff.first); - std::string const tokenIdStr = token->CreateSymbolKey(diff.first); - - if(tokenIdStr == tokenFilter) { - hasToken = true; - break; - } - } - - if(!hasToken) { - return true; // continue - } + txs.insert(value.txid); } + --limit; + } - if(noRewards) { - if(value.category == static_cast(CustomTxType::NonTxRewards)) { - return true; // continue - } - } + return limit != 0; + }, { account, startBlock, std::numeric_limits::max() }); - ret.push_back(accounthistoryToJSON(key, value)); - if (shouldSearchInWallet) { - txs.insert(txid); - } - return --limit != 0; - }, startKey); + if (!limit) { + return ret; } if (shouldSearchInWallet) { @@ -3284,7 +3208,7 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { const auto& txOrdered = pwallet->wtxOrdered; - for (auto it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { + for (auto it = txOrdered.rbegin(); limit != 0 && it != txOrdered.rend(); ++it) { CWalletTx *const pwtx = (*it).second; if(pwtx->IsCoinBase()) { @@ -3296,22 +3220,46 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { } pwtx->GetAmounts(listReceived, listSent, nFee, filter); const auto index = LookupBlockIndex(pwtx->hashBlock); - for (auto& sent : listSent) { - if (IsValidDestination(destination) && destination != sent.destination) { + for (auto it = listSent.begin(); limit != 0 && it != listSent.end(); ++it) { + if (IsValidDestination(destination) && destination != it->destination) { continue; } - sent.amount = -sent.amount; - ret.push_back(outputEntryToJSON(sent, index->nHeight, pwtx->hashBlock, txid, "sent")); + it->amount = -(it->amount); + ret.push_back(outputEntryToJSON(*it, index->nHeight, txid, "sent")); + --limit; } - for (auto& recv : listReceived) { - if (IsValidDestination(destination) && destination != recv.destination) { + for (auto it = listReceived.begin(); limit != 0 && it != listReceived.end(); ++it) { + if (IsValidDestination(destination) && destination != it->destination) { continue; } - ret.push_back(outputEntryToJSON(recv, index->nHeight, pwtx->hashBlock, txid, "receive")); + ret.push_back(outputEntryToJSON(*it, index->nHeight, txid, "receive")); + --limit; } } } + if (noRewards || !limit) { + return ret; + } + + pcustomcsview->ForEachRewardHistory([&](RewardHistoryKey const & key, CLazySerialize valueLazy) { + if (preCondition(key.blockHeight, key.owner)) + return true; // continue + + auto value = valueLazy.get(); + + if(!tokenFilter.empty() && !hasToken(value.diff)) { + return true; + } + + if (isForMe(key.owner)) { + ret.push_back(rewardhistoryToJSON(key, value)); + --limit; + } + + return limit != 0; + }, { account, startBlock, std::numeric_limits::max() }); + return ret; } diff --git a/src/masternodes/poolpairs.h b/src/masternodes/poolpairs.h index f5e9062963..5127678b3a 100644 --- a/src/masternodes/poolpairs.h +++ b/src/masternodes/poolpairs.h @@ -230,6 +230,21 @@ struct PoolShareKey { } }; +enum class RewardType : uint8_t +{ + Commission = 128, + PoolReward = 129, +}; + +inline std::string RewardToString(RewardType type) +{ + switch(type) { + case RewardType::Commission: return "Commission"; + case RewardType::PoolReward: return "PoolReward"; + } + return "Unknown"; +} + class CPoolPairView : public virtual CStorageView { public: @@ -253,7 +268,7 @@ class CPoolPairView : public virtual CStorageView } /// @attention it throws (at least for debug), cause errors are critical! - CAmount DistributeRewards(CAmount yieldFarming, std::function onGetBalance, std::function onTransfer, bool newRewardCalc = false) { + CAmount DistributeRewards(CAmount yieldFarming, std::function onGetBalance, std::function onTransfer, bool newRewardCalc = false) { uint32_t const PRECISION = 10000; // (== 100%) just searching the way to avoid arith256 inflating CAmount totalDistributed = 0; @@ -285,13 +300,13 @@ class CPoolPairView : public virtual CStorageView feeA = pool.blockCommissionA * liqWeight / PRECISION; } distributedFeeA += feeA; - onTransfer(provider, {pool.idTokenA, feeA}); //can throw + onTransfer(provider, poolId, uint8_t(RewardType::Commission), {pool.idTokenA, feeA}); //can throw CAmount feeB = static_cast((arith_uint256(pool.blockCommissionB) * arith_uint256(liquidity) / arith_uint256(pool.totalLiquidity)).GetLow64()); if (!newRewardCalc) { feeB = pool.blockCommissionB * liqWeight / PRECISION; } distributedFeeB += feeB; - onTransfer(provider, {pool.idTokenB, feeB}); //can throw + onTransfer(provider, poolId, uint8_t(RewardType::Commission), {pool.idTokenB, feeB}); //can throw } // distribute yield farming @@ -301,7 +316,7 @@ class CPoolPairView : public virtual CStorageView providerReward = poolReward * liqWeight / PRECISION; } if (providerReward) { - onTransfer(provider, {DCT_ID{0}, providerReward}); //can throw + onTransfer(provider, poolId, uint8_t(RewardType::PoolReward), {DCT_ID{0}, providerReward}); //can throw totalDistributed += providerReward; } } diff --git a/src/test/liquidity_tests.cpp b/src/test/liquidity_tests.cpp index a8e5b84b23..c7a4f7c81c 100644 --- a/src/test/liquidity_tests.cpp +++ b/src/test/liquidity_tests.cpp @@ -315,7 +315,7 @@ BOOST_AUTO_TEST_CASE(math_rewards) [&cache] (CScript const & owner, DCT_ID tokenID) { return cache.GetBalance(owner, tokenID); }, - [&cache] (CScript const & to, CTokenAmount amount) { + [&cache] (CScript const & to, DCT_ID poolID, uint8_t type, CTokenAmount amount) { return cache.AddBalance(to, amount); } ); diff --git a/src/validation.cpp b/src/validation.cpp index 17e6eca8bc..aaba18dc40 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2266,7 +2266,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl { // old data pruning and other (some processing made for the whole block) // make all changes to the new cache/snapshot to make it possible to take a diff later: - CAccountsHistoryStorage cache(mnview, static_cast(pindex->nHeight), std::numeric_limits::max(), uint256(), uint8_t(CustomTxType::NonTxRewards)); + CRewardsHistoryStorage cache(mnview, static_cast(pindex->nHeight)); // cache.CallYourInterblockProcessingsHere(); @@ -2283,8 +2283,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl [&cache] (CScript const & owner, DCT_ID tokenID) { return cache.GetBalance(owner, tokenID); }, - [&cache, &block] (CScript const & to, CTokenAmount amount) { - auto res = cache.AddBalance(to, amount); + [&cache, &block] (CScript const & to, DCT_ID poolID, uint8_t type, CTokenAmount amount) { + auto res = cache.AddBalance(to, poolID, type, amount); if (!res.ok) throw std::runtime_error(strprintf("Pool rewards: can't update balance of %s: %s, Block %ld (%s)", to.GetHex(), res.msg, block.height, block.GetHash().ToString())); return res; From aaa0d0d9be5d0ed6227b5f30e2422995d479dde7 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Wed, 9 Dec 2020 08:04:06 +0200 Subject: [PATCH 4/9] Rename PoolReward -> Rewards Signed-off-by: Anthony Fieroni --- src/masternodes/poolpairs.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/masternodes/poolpairs.h b/src/masternodes/poolpairs.h index 5127678b3a..3a94902a01 100644 --- a/src/masternodes/poolpairs.h +++ b/src/masternodes/poolpairs.h @@ -233,14 +233,14 @@ struct PoolShareKey { enum class RewardType : uint8_t { Commission = 128, - PoolReward = 129, + Rewards = 129, }; inline std::string RewardToString(RewardType type) { switch(type) { case RewardType::Commission: return "Commission"; - case RewardType::PoolReward: return "PoolReward"; + case RewardType::Rewards: return "Rewards"; } return "Unknown"; } @@ -316,7 +316,7 @@ class CPoolPairView : public virtual CStorageView providerReward = poolReward * liqWeight / PRECISION; } if (providerReward) { - onTransfer(provider, poolId, uint8_t(RewardType::PoolReward), {DCT_ID{0}, providerReward}); //can throw + onTransfer(provider, poolId, uint8_t(RewardType::Rewards), {DCT_ID{0}, providerReward}); //can throw totalDistributed += providerReward; } } From 4cf9a2f8397e48dd000541092bd06384a81d026b Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Wed, 9 Dec 2020 08:16:02 +0200 Subject: [PATCH 5/9] Add block hash to account history Signed-off-by: Anthony Fieroni --- src/masternodes/mn_rpc.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/masternodes/mn_rpc.cpp b/src/masternodes/mn_rpc.cpp index bb2972b6e2..60a8eb4402 100644 --- a/src/masternodes/mn_rpc.cpp +++ b/src/masternodes/mn_rpc.cpp @@ -3005,6 +3005,9 @@ UniValue accounthistoryToJSON(AccountHistoryKey const & key, AccountHistoryValue obj.pushKV("owner", ScriptToString(key.owner)); obj.pushKV("blockHeight", (uint64_t) key.blockHeight); + if (auto block = ::ChainActive()[key.blockHeight]) { + obj.pushKV("blockHash", block->GetBlockHash().GetHex()); + } obj.pushKV("type", ToString(CustomTxCodeToType(value.category))); obj.pushKV("txn", (uint64_t) key.txn); obj.pushKV("txid", value.txid.ToString()); @@ -3016,17 +3019,21 @@ UniValue rewardhistoryToJSON(RewardHistoryKey const & key, RewardHistoryValue co UniValue obj(UniValue::VOBJ); obj.pushKV("owner", ScriptToString(key.owner)); obj.pushKV("blockHeight", (uint64_t) key.blockHeight); + if (auto block = ::ChainActive()[key.blockHeight]) { + obj.pushKV("blockHash", block->GetBlockHash().GetHex()); + } obj.pushKV("type", RewardToString(RewardType(value.category))); obj.pushKV("poolID", key.poolID.ToString()); obj.pushKV("amounts", AmountsToJSON(value.diff)); return obj; } -UniValue outputEntryToJSON(COutputEntry const & entry, uint32_t height, uint256 const & txid, std::string const & type) { +UniValue outputEntryToJSON(COutputEntry const & entry, uint32_t height, uint256 const & hashBlock, uint256 const & txid, std::string const & type) { UniValue obj(UniValue::VOBJ); obj.pushKV("owner", EncodeDestination(entry.destination)); obj.pushKV("blockHeight", (uint64_t)height); + obj.pushKV("blockHash", hashBlock.GetHex()); obj.pushKV("type", type); obj.pushKV("txn", (uint64_t) entry.vout); obj.pushKV("txid", txid.ToString()); @@ -3225,14 +3232,14 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { continue; } it->amount = -(it->amount); - ret.push_back(outputEntryToJSON(*it, index->nHeight, txid, "sent")); + ret.push_back(outputEntryToJSON(*it, index->nHeight, pwtx->hashBlock, txid, "sent")); --limit; } for (auto it = listReceived.begin(); limit != 0 && it != listReceived.end(); ++it) { if (IsValidDestination(destination) && destination != it->destination) { continue; } - ret.push_back(outputEntryToJSON(*it, index->nHeight, txid, "receive")); + ret.push_back(outputEntryToJSON(*it, index->nHeight, pwtx->hashBlock, txid, "receive")); --limit; } } From 8be90cd6aad9c88571ae64cee5972acefeaaafb5 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Thu, 10 Dec 2020 14:59:55 +0200 Subject: [PATCH 6/9] Filter unknown addresses Signed-off-by: Anthony Fieroni --- src/masternodes/mn_rpc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/masternodes/mn_rpc.cpp b/src/masternodes/mn_rpc.cpp index 60a8eb4402..8970c0ca54 100644 --- a/src/masternodes/mn_rpc.cpp +++ b/src/masternodes/mn_rpc.cpp @@ -3228,7 +3228,7 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { pwtx->GetAmounts(listReceived, listSent, nFee, filter); const auto index = LookupBlockIndex(pwtx->hashBlock); for (auto it = listSent.begin(); limit != 0 && it != listSent.end(); ++it) { - if (IsValidDestination(destination) && destination != it->destination) { + if (!IsValidDestination(it->destination) || (IsValidDestination(destination) && destination != it->destination)) { continue; } it->amount = -(it->amount); @@ -3236,7 +3236,7 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { --limit; } for (auto it = listReceived.begin(); limit != 0 && it != listReceived.end(); ++it) { - if (IsValidDestination(destination) && destination != it->destination) { + if (!IsValidDestination(it->destination) || (IsValidDestination(destination) && destination != it->destination)) { continue; } ret.push_back(outputEntryToJSON(*it, index->nHeight, pwtx->hashBlock, txid, "receive")); From b502c3c2287536b7b689dc7954353fd7a1702fcb Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Mon, 14 Dec 2020 17:19:39 +0200 Subject: [PATCH 7/9] Better find old history format Signed-off-by: Anthony Fieroni --- src/init.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index ac97cea6d6..db358addcf 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1586,18 +1586,24 @@ bool AppInitMain(InitInterfaces& interfaces) pcustomcsview.reset(); pcustomcsview = MakeUnique(*pcustomcsDB.get()); if (!fReset && gArgs.GetBoolArg("-acindex", false)) { - bool hasAccountHistory = false, hasRewardHistory = false; - pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const &, CLazySerialize) { - hasAccountHistory = true; - return false; - }); + bool hasRewardHistory = false; pcustomcsview->ForEachRewardHistory([&](RewardHistoryKey const &, CLazySerialize) { hasRewardHistory = true; return false; }); - if (hasAccountHistory && !hasRewardHistory) { - strLoadError = _("Account history needs rebuild").translated; - break; + if (!hasRewardHistory) { + bool hasOldAccountHistory = false; + pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, CLazySerialize) { + if (key.txn == std::numeric_limits::max()) { + hasOldAccountHistory = true; + return false; + } + return true; + }, { {}, 0, std::numeric_limits::max() }); + if (hasOldAccountHistory) { + strLoadError = _("Account history needs rebuild").translated; + break; + } } } From c03873f427ea103c55ae5ec2bf246a6c8bb22a26 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Tue, 15 Dec 2020 10:01:37 +0200 Subject: [PATCH 8/9] Add blocktime to account history Signed-off-by: Anthony Fieroni --- src/masternodes/mn_rpc.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/masternodes/mn_rpc.cpp b/src/masternodes/mn_rpc.cpp index 8970c0ca54..a445ebef3b 100644 --- a/src/masternodes/mn_rpc.cpp +++ b/src/masternodes/mn_rpc.cpp @@ -3007,6 +3007,7 @@ UniValue accounthistoryToJSON(AccountHistoryKey const & key, AccountHistoryValue obj.pushKV("blockHeight", (uint64_t) key.blockHeight); if (auto block = ::ChainActive()[key.blockHeight]) { obj.pushKV("blockHash", block->GetBlockHash().GetHex()); + obj.pushKV("blockTime", block->GetBlockTime()); } obj.pushKV("type", ToString(CustomTxCodeToType(value.category))); obj.pushKV("txn", (uint64_t) key.txn); @@ -3021,6 +3022,7 @@ UniValue rewardhistoryToJSON(RewardHistoryKey const & key, RewardHistoryValue co obj.pushKV("blockHeight", (uint64_t) key.blockHeight); if (auto block = ::ChainActive()[key.blockHeight]) { obj.pushKV("blockHash", block->GetBlockHash().GetHex()); + obj.pushKV("blockTime", block->GetBlockTime()); } obj.pushKV("type", RewardToString(RewardType(value.category))); obj.pushKV("poolID", key.poolID.ToString()); @@ -3028,12 +3030,13 @@ UniValue rewardhistoryToJSON(RewardHistoryKey const & key, RewardHistoryValue co return obj; } -UniValue outputEntryToJSON(COutputEntry const & entry, uint32_t height, uint256 const & hashBlock, uint256 const & txid, std::string const & type) { +UniValue outputEntryToJSON(COutputEntry const & entry, CBlockIndex const * index, uint256 const & txid, std::string const & type) { UniValue obj(UniValue::VOBJ); obj.pushKV("owner", EncodeDestination(entry.destination)); - obj.pushKV("blockHeight", (uint64_t)height); - obj.pushKV("blockHash", hashBlock.GetHex()); + obj.pushKV("blockHeight", index->height); + obj.pushKV("blockHash", index->GetBlockHash().GetHex()); + obj.pushKV("blockTime", index->GetBlockTime()); obj.pushKV("type", type); obj.pushKV("txn", (uint64_t) entry.vout); obj.pushKV("txid", txid.ToString()); @@ -3232,14 +3235,14 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { continue; } it->amount = -(it->amount); - ret.push_back(outputEntryToJSON(*it, index->nHeight, pwtx->hashBlock, txid, "sent")); + ret.push_back(outputEntryToJSON(*it, index, txid, "sent")); --limit; } for (auto it = listReceived.begin(); limit != 0 && it != listReceived.end(); ++it) { if (!IsValidDestination(it->destination) || (IsValidDestination(destination) && destination != it->destination)) { continue; } - ret.push_back(outputEntryToJSON(*it, index->nHeight, pwtx->hashBlock, txid, "receive")); + ret.push_back(outputEntryToJSON(*it, index, txid, "receive")); --limit; } } From fede87c8a7b6e13456d168233042e57b05bf71c7 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Tue, 15 Dec 2020 12:11:23 +0200 Subject: [PATCH 9/9] Add accounthistorycount rpc method Signed-off-by: Anthony Fieroni --- src/init.cpp | 2 +- src/masternodes/mn_rpc.cpp | 139 +++++++++++++++++++++++++++++++++++++ src/rpc/client.cpp | 1 + 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index db358addcf..1569a41729 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -415,7 +415,7 @@ void SetupServerArgs() hidden_args.emplace_back("-sysperms"); #endif gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-acindex", strprintf("Maintain a full account history index, tracking all accounts balances changes. Used by the listaccounthistory rpc call (default: %u)", false), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-acindex", strprintf("Maintain a full account history index, tracking all accounts balances changes. Used by the listaccounthistory and accounthistorycount rpc calls (default: %u)", false), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-blockfilterindex=", strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) + " If is not supplied or if = 1, indexes for all known types are enabled.", diff --git a/src/masternodes/mn_rpc.cpp b/src/masternodes/mn_rpc.cpp index a445ebef3b..b39eab2fdd 100644 --- a/src/masternodes/mn_rpc.cpp +++ b/src/masternodes/mn_rpc.cpp @@ -3216,6 +3216,8 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { ExtractDestination(owner, destination); } + LOCK(pwallet->cs_wallet); + const auto& txOrdered = pwallet->wtxOrdered; for (auto it = txOrdered.rbegin(); limit != 0 && it != txOrdered.rend(); ++it) { @@ -3273,6 +3275,142 @@ UniValue listaccounthistory(const JSONRPCRequest& request) { return ret; } +UniValue accounthistorycount(const JSONRPCRequest& request) { + CWallet* const pwallet = GetWallet(request); + RPCHelpMan{"accounthistorycount", + "\nReturns count of account history.\n", + { + {"owner", RPCArg::Type::STR, RPCArg::Optional::OMITTED, + "Single account ID (CScript or address) or reserved words: \"mine\" - to list history for all owned accounts or \"all\" to list whole DB (default = \"mine\")."}, + + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"no_rewards", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED, "Filter out rewards"}, + }, + }, + }, + RPCResult{ + "count (int) Count of account history\n" + }, + RPCExamples{ + HelpExampleCli("accounthistorycount", "all '{no_rewards: true}'") + + HelpExampleRpc("accounthistorycount", "") + }, + }.Check(request); + + std::string accounts = "mine"; + if (request.params.size() > 0) { + accounts = request.params[0].getValStr(); + } + + bool noRewards = false; + + if (request.params.size() > 1) { + UniValue optionsObj = request.params[1].get_obj(); + RPCTypeCheckObj(optionsObj, + { + {"no_rewards", UniValueType(UniValue::VBOOL)}, + }, true, true); + + noRewards = optionsObj["no_rewards"].get_bool(); + } + + LOCK(cs_main); + UniValue ret(UniValue::VARR); + + uint64_t count = 0; + std::set txs; + + CScript owner; + std::function isForMe = [](CScript const&) { return true; }; + + if (accounts == "mine") { + isForMe = [pwallet](CScript const & owner) { + return IsMine(*pwallet, owner) == ISMINE_SPENDABLE; + }; + } else if (accounts != "all") { + owner = DecodeScript(accounts); + } + + pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, CLazySerialize valueLazy) { + + if (!owner.empty() && owner != key.owner) { + return false; + } + + if (isForMe(key.owner)) { + ++count; + txs.insert(valueLazy.get().txid); + } + + return true; + }, {owner, 0, 0} ); + + { + CAmount nFee; + std::list listSent; + std::list listReceived; + + LOCK(pwallet->cs_wallet); + + const auto& txOrdered = pwallet->wtxOrdered; + + for (const auto& tx : txOrdered) { + + auto pwtx = tx.second; + + if(pwtx->IsCoinBase()) { + continue; + } + + CTxDestination destination; + if (!owner.empty()) { + ExtractDestination(owner, destination); + } + + const auto& txid = pwtx->GetHash(); + if (txs.count(txid)) { + continue; + } + + pwtx->GetAmounts(listReceived, listSent, nFee, ISMINE_ALL_USED); + + for (const auto& sent : listSent) { + if (!IsValidDestination(sent.destination) || (IsValidDestination(destination) && destination != sent.destination)) { + continue; + } + ++count; + } + + for (const auto& recv : listReceived) { + if (!IsValidDestination(recv.destination) || (IsValidDestination(destination) && destination != recv.destination)) { + continue; + } + ++count; + } + } + } + + if (noRewards) { + return count; + } + + pcustomcsview->ForEachRewardHistory([&](RewardHistoryKey const & key, CLazySerialize) { + + if (!owner.empty() && owner != key.owner) { + return false; + } + + if (isForMe(key.owner)) { + ++count; + } + + return true; + }, {owner, 0, 0} ); + + return count; +} + UniValue listcommunitybalances(const JSONRPCRequest& request) { RPCHelpMan{"listcommunitybalances", "\nReturns information about all community balances.\n", @@ -3604,6 +3742,7 @@ static const CRPCCommand commands[] = {"poolpair", "listpoolshares", &listpoolshares, {"pagination", "verbose", "is_mine_only"}}, {"poolpair", "testpoolswap", &testpoolswap, {"metadata"}}, {"accounts", "listaccounthistory", &listaccounthistory, {"owner", "options"}}, + {"accounts", "accounthistorycount", &accounthistorycount, {"owner", "options"}}, {"accounts", "listcommunitybalances", &listcommunitybalances, {}}, {"blockchain", "setgov", &setgov, {"variables", "inputs"}}, {"blockchain", "getgov", &getgov, {"name"}}, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index d64e4d6e40..3e8f7b724d 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -238,6 +238,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listpoolshares", 2, "is_mine_only" }, { "listaccounthistory", 1, "options" }, + { "accounthistorycount", 1, "options" }, { "setgov", 0, "variables" }, { "setgov", 1, "inputs" },