From 080014d0c1d384cf41218fb0656751866b5c6405 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Tue, 14 Feb 2023 14:34:32 +0000 Subject: [PATCH 1/7] Threaded getburninfo --- src/masternodes/accountshistory.cpp | 6 + src/masternodes/accountshistory.h | 4 + src/masternodes/rpc_accounts.cpp | 168 ++++++++++++++++++---------- src/masternodes/validation.h | 2 + 4 files changed, 122 insertions(+), 58 deletions(-) diff --git a/src/masternodes/accountshistory.cpp b/src/masternodes/accountshistory.cpp index 298af8cb64..a68c68ed91 100644 --- a/src/masternodes/accountshistory.cpp +++ b/src/masternodes/accountshistory.cpp @@ -57,6 +57,12 @@ void CAccountsHistoryView::ForEachAccountHistory( {height, owner, txn}); } +void CAccountsHistoryView::ForEachAccountHistoryNew( + std::function callback, + const AccountHistoryKeyNew &start) { + ForEach(callback, start); +} + std::optional CAccountsHistoryView::ReadAccountHistory(const AccountHistoryKey &key) const { return ReadBy(key); } diff --git a/src/masternodes/accountshistory.h b/src/masternodes/accountshistory.h index 6c3f423039..1375d129c1 100644 --- a/src/masternodes/accountshistory.h +++ b/src/masternodes/accountshistory.h @@ -29,6 +29,10 @@ class CAccountsHistoryView : public virtual CStorageView { const CScript &owner = {}, uint32_t height = std::numeric_limits::max(), uint32_t txn = std::numeric_limits::max()); + void ForEachAccountHistoryNew(std::function callback, + const AccountHistoryKeyNew &start = {std::numeric_limits::max(), + {}, + std::numeric_limits::max()}); // tags struct ByAccountHistoryKey { diff --git a/src/masternodes/rpc_accounts.cpp b/src/masternodes/rpc_accounts.cpp index 5cdfacc49e..05f2a973bc 100644 --- a/src/masternodes/rpc_accounts.cpp +++ b/src/masternodes/rpc_accounts.cpp @@ -1,6 +1,17 @@ #include #include #include +#include + +struct BalanceResults { + CAmount burntDFI{}; + CAmount burntFee{}; + CAmount auctionFee{}; + CBalances burntTokens; + CBalances nonConsortiumTokens; + CBalances dexfeeburn; + CBalances paybackFee; +}; std::string tokenAmountString(CTokenAmount const& amount, AmountFormat format = AmountFormat::Symbol) { const auto token = pcustomcsview->GetToken(amount.nTokenId); @@ -1991,67 +2002,108 @@ UniValue getburninfo(const JSONRPCRequest& request) { } } - auto calculateBurnAmounts = [&](AccountHistoryKey const& key, AccountHistoryValue value) { - // UTXO burn - if (value.category == uint8_t(CustomTxType::None)) { - for (auto const & diff : value.diff) { - burntDFI += diff.second; - } - return true; - } - // Fee burn - if (value.category == uint8_t(CustomTxType::CreateMasternode) - || value.category == uint8_t(CustomTxType::CreateToken) - || value.category == uint8_t(CustomTxType::Vault) - || value.category == uint8_t(CustomTxType::CreateCfp) - || value.category == uint8_t(CustomTxType::CreateVoc)) { - for (auto const & diff : value.diff) { - burntFee += diff.second; - } - return true; - } - // withdraw burn - if (value.category == uint8_t(CustomTxType::PaybackLoan) - || value.category == uint8_t(CustomTxType::PaybackLoanV2) - || value.category == uint8_t(CustomTxType::PaybackWithCollateral)) { - for (const auto& [id, amount] : value.diff) { - paybackFee.Add({id, amount}); - } - return true; - } - // auction burn - if (value.category == uint8_t(CustomTxType::AuctionBid)) { - for (auto const & diff : value.diff) { - auctionFee += diff.second; - } - return true; - } - // dex fee burn - if (value.category == uint8_t(CustomTxType::PoolSwap) - || value.category == uint8_t(CustomTxType::PoolSwapV2)) { - for (auto const & diff : value.diff) { - dexfeeburn.Add({diff.first, diff.second}); - } - return true; - } + const auto nWorkers = RewardConsolidationWorkersCount(); + const auto currentHeight = ChainActive().Height(); + const auto chunks = currentHeight / nWorkers; + const auto chunksRemainder = currentHeight % nWorkers; - // token burn with burnToken tx - if (value.category == uint8_t(CustomTxType::BurnToken)) - { - for (auto const & diff : value.diff) { - nonConsortiumTokens.Add({diff.first, diff.second}); - } - return true; - } + std::vector> workerResults; + std::vector threads; - // Token burn - for (auto const & diff : value.diff) { - burntTokens.Add({diff.first, diff.second}); - } - return true; - }; + for (size_t i{}; i < nWorkers; ++i) { + const uint32_t startHeight = i + 1 == nWorkers ? (i + 1) * chunks + chunksRemainder : (i + 1) * chunks; + const uint32_t stopHeight = i + 1 == nWorkers ? startHeight - chunks - chunksRemainder : startHeight - chunks; + + auto result = std::make_shared(); + workerResults.push_back(result); - burnView->ForEachAccountHistory(calculateBurnAmounts); + threads.emplace_back([result, startHeight, stopHeight]{ + burnView->ForEachAccountHistoryNew([result, startHeight, stopHeight](const AccountHistoryKeyNew &key, const AccountHistoryValue &value) { + + // Stop on chunk range for worker + if (key.blockHeight == stopHeight) { + return false; + } + + // UTXO burn + if (value.category == uint8_t(CustomTxType::None)) { + for (auto const & diff : value.diff) { + result->burntDFI += diff.second; + } + return true; + } + + // Fee burn + if (value.category == uint8_t(CustomTxType::CreateMasternode) + || value.category == uint8_t(CustomTxType::CreateToken) + || value.category == uint8_t(CustomTxType::Vault) + || value.category == uint8_t(CustomTxType::CreateCfp) + || value.category == uint8_t(CustomTxType::CreateVoc)) { + for (auto const & diff : value.diff) { + result->burntFee += diff.second; + } + return true; + } + + // withdraw burn + if (value.category == uint8_t(CustomTxType::PaybackLoan) + || value.category == uint8_t(CustomTxType::PaybackLoanV2) + || value.category == uint8_t(CustomTxType::PaybackWithCollateral)) { + for (const auto& [id, amount] : value.diff) { + result->paybackFee.Add({id, amount}); + } + return true; + } + + // auction burn + if (value.category == uint8_t(CustomTxType::AuctionBid)) { + for (auto const & diff : value.diff) { + result->auctionFee += diff.second; + } + return true; + } + + // dex fee burn + if (value.category == uint8_t(CustomTxType::PoolSwap) + || value.category == uint8_t(CustomTxType::PoolSwapV2)) { + for (auto const & diff : value.diff) { + result->dexfeeburn.Add({diff.first, diff.second}); + } + return true; + } + + // token burn with burnToken tx + if (value.category == uint8_t(CustomTxType::BurnToken)) + { + for (auto const & diff : value.diff) { + result->nonConsortiumTokens.Add({diff.first, diff.second}); + } + return true; + } + + // Token burn + for (auto const & diff : value.diff) { + result->burntTokens.Add({diff.first, diff.second}); + } + + return true; + }, {startHeight, {}, std::numeric_limits::max()}); + }); + } + + for (std::thread& t : threads) { + t.join(); + } + + for (const auto &result : workerResults) { + burntDFI += result->burntDFI; + burntFee += result->burntFee; + auctionFee += result->auctionFee; + burntTokens.AddBalances(result->burntTokens.balances); + nonConsortiumTokens.AddBalances(result->nonConsortiumTokens.balances); + dexfeeburn.AddBalances(result->dexfeeburn.balances); + paybackFee.AddBalances(result->paybackFee.balances); + } CDataStructureV0 liveKey = {AttributeTypes::Live, ParamIDs::Economy, EconomyKeys::ConsortiumMinted}; auto balances = attributes->GetValue(liveKey, CConsortiumGlobalMinted{}); diff --git a/src/masternodes/validation.h b/src/masternodes/validation.h index 9e2f986a2a..5aa49e5963 100644 --- a/src/masternodes/validation.h +++ b/src/masternodes/validation.h @@ -20,4 +20,6 @@ using CreationTxs = std::map CollectAuctionBatches(const CCollateralLoans& collLoan, const TAmounts& collBalances, const TAmounts& loanBalances); +size_t RewardConsolidationWorkersCount(); + #endif // DEFI_MASTERNODES_VALIDATION_H From e7391be9bf667d61f380605ab4e35d648373a4b6 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 15 Feb 2023 07:09:10 +0000 Subject: [PATCH 2/7] Use ForEachAccountHistory --- src/masternodes/accountshistory.cpp | 6 ------ src/masternodes/accountshistory.h | 4 ---- src/masternodes/rpc_accounts.cpp | 6 +++--- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/masternodes/accountshistory.cpp b/src/masternodes/accountshistory.cpp index a68c68ed91..298af8cb64 100644 --- a/src/masternodes/accountshistory.cpp +++ b/src/masternodes/accountshistory.cpp @@ -57,12 +57,6 @@ void CAccountsHistoryView::ForEachAccountHistory( {height, owner, txn}); } -void CAccountsHistoryView::ForEachAccountHistoryNew( - std::function callback, - const AccountHistoryKeyNew &start) { - ForEach(callback, start); -} - std::optional CAccountsHistoryView::ReadAccountHistory(const AccountHistoryKey &key) const { return ReadBy(key); } diff --git a/src/masternodes/accountshistory.h b/src/masternodes/accountshistory.h index 1375d129c1..6c3f423039 100644 --- a/src/masternodes/accountshistory.h +++ b/src/masternodes/accountshistory.h @@ -29,10 +29,6 @@ class CAccountsHistoryView : public virtual CStorageView { const CScript &owner = {}, uint32_t height = std::numeric_limits::max(), uint32_t txn = std::numeric_limits::max()); - void ForEachAccountHistoryNew(std::function callback, - const AccountHistoryKeyNew &start = {std::numeric_limits::max(), - {}, - std::numeric_limits::max()}); // tags struct ByAccountHistoryKey { diff --git a/src/masternodes/rpc_accounts.cpp b/src/masternodes/rpc_accounts.cpp index 05f2a973bc..0b92fcd431 100644 --- a/src/masternodes/rpc_accounts.cpp +++ b/src/masternodes/rpc_accounts.cpp @@ -2018,10 +2018,10 @@ UniValue getburninfo(const JSONRPCRequest& request) { workerResults.push_back(result); threads.emplace_back([result, startHeight, stopHeight]{ - burnView->ForEachAccountHistoryNew([result, startHeight, stopHeight](const AccountHistoryKeyNew &key, const AccountHistoryValue &value) { + burnView->ForEachAccountHistory([result, stopHeight](const AccountHistoryKey &key, const AccountHistoryValue &value) { // Stop on chunk range for worker - if (key.blockHeight == stopHeight) { + if (key.blockHeight <= stopHeight) { return false; } @@ -2087,7 +2087,7 @@ UniValue getburninfo(const JSONRPCRequest& request) { } return true; - }, {startHeight, {}, std::numeric_limits::max()}); + }, {}, startHeight, std::numeric_limits::max()); }); } From e5853eb3a84a0723b681b4e7197a7d9aca020ca0 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 15 Feb 2023 07:23:25 +0000 Subject: [PATCH 3/7] Use global not locally defined reference to global --- src/masternodes/rpc_accounts.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/masternodes/rpc_accounts.cpp b/src/masternodes/rpc_accounts.cpp index 0b92fcd431..7228503a5c 100644 --- a/src/masternodes/rpc_accounts.cpp +++ b/src/masternodes/rpc_accounts.cpp @@ -1969,7 +1969,6 @@ UniValue getburninfo(const JSONRPCRequest& request) { auto fortCanningHeight = Params().GetConsensus().FortCanningHeight; auto burnAddress = Params().GetConsensus().burnAddress; auto view = *pcustomcsview; - auto &burnView = pburnHistoryDB; auto attributes = view.GetAttributes(); if (attributes) { @@ -2018,7 +2017,7 @@ UniValue getburninfo(const JSONRPCRequest& request) { workerResults.push_back(result); threads.emplace_back([result, startHeight, stopHeight]{ - burnView->ForEachAccountHistory([result, stopHeight](const AccountHistoryKey &key, const AccountHistoryValue &value) { + pburnHistoryDB->ForEachAccountHistory([result, stopHeight](const AccountHistoryKey &key, const AccountHistoryValue &value) { // Stop on chunk range for worker if (key.blockHeight <= stopHeight) { From 3ed4253cf0c716fba95f046895c48e2134fcd22e Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 15 Feb 2023 07:43:32 +0000 Subject: [PATCH 4/7] Use Boost ASIO --- src/masternodes/rpc_accounts.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/masternodes/rpc_accounts.cpp b/src/masternodes/rpc_accounts.cpp index 7228503a5c..93982fa683 100644 --- a/src/masternodes/rpc_accounts.cpp +++ b/src/masternodes/rpc_accounts.cpp @@ -3,6 +3,8 @@ #include #include +#include + struct BalanceResults { CAmount burntDFI{}; CAmount burntFee{}; @@ -2007,7 +2009,7 @@ UniValue getburninfo(const JSONRPCRequest& request) { const auto chunksRemainder = currentHeight % nWorkers; std::vector> workerResults; - std::vector threads; + boost::asio::thread_pool workerPool{nWorkers}; for (size_t i{}; i < nWorkers; ++i) { const uint32_t startHeight = i + 1 == nWorkers ? (i + 1) * chunks + chunksRemainder : (i + 1) * chunks; @@ -2016,7 +2018,7 @@ UniValue getburninfo(const JSONRPCRequest& request) { auto result = std::make_shared(); workerResults.push_back(result); - threads.emplace_back([result, startHeight, stopHeight]{ + boost::asio::post(workerPool, [result, startHeight, stopHeight]{ pburnHistoryDB->ForEachAccountHistory([result, stopHeight](const AccountHistoryKey &key, const AccountHistoryValue &value) { // Stop on chunk range for worker @@ -2090,9 +2092,7 @@ UniValue getburninfo(const JSONRPCRequest& request) { }); } - for (std::thread& t : threads) { - t.join(); - } + workerPool.join(); for (const auto &result : workerResults) { burntDFI += result->burntDFI; From 95e33cfe7376a9e7fcce54b8641845da471a5886 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 15 Feb 2023 08:13:28 +0000 Subject: [PATCH 5/7] Make setting of heights easier to read --- src/masternodes/rpc_accounts.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/masternodes/rpc_accounts.cpp b/src/masternodes/rpc_accounts.cpp index 93982fa683..ce5b40b70e 100644 --- a/src/masternodes/rpc_accounts.cpp +++ b/src/masternodes/rpc_accounts.cpp @@ -2012,8 +2012,14 @@ UniValue getburninfo(const JSONRPCRequest& request) { boost::asio::thread_pool workerPool{nWorkers}; for (size_t i{}; i < nWorkers; ++i) { - const uint32_t startHeight = i + 1 == nWorkers ? (i + 1) * chunks + chunksRemainder : (i + 1) * chunks; - const uint32_t stopHeight = i + 1 == nWorkers ? startHeight - chunks - chunksRemainder : startHeight - chunks; + uint32_t startHeight{}, stopHeight{}; + if (i + 1 == nWorkers) { + startHeight = (i + 1) * chunks + chunksRemainder; + stopHeight = startHeight - chunks - chunksRemainder; + } else { + startHeight = (i + 1) * chunks; + stopHeight = nWorkers ? : startHeight - chunks; + } auto result = std::make_shared(); workerResults.push_back(result); From d335af6198862dd1cb43e469c9253770d0ae986d Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 15 Feb 2023 08:21:49 +0000 Subject: [PATCH 6/7] Set initial values, then last chunk on last worker. --- src/masternodes/rpc_accounts.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/masternodes/rpc_accounts.cpp b/src/masternodes/rpc_accounts.cpp index ce5b40b70e..4c831bfb5e 100644 --- a/src/masternodes/rpc_accounts.cpp +++ b/src/masternodes/rpc_accounts.cpp @@ -2012,13 +2012,12 @@ UniValue getburninfo(const JSONRPCRequest& request) { boost::asio::thread_pool workerPool{nWorkers}; for (size_t i{}; i < nWorkers; ++i) { - uint32_t startHeight{}, stopHeight{}; + uint32_t startHeight = (i + 1) * chunks; + uint32_t stopHeight = startHeight - chunks; + if (i + 1 == nWorkers) { startHeight = (i + 1) * chunks + chunksRemainder; stopHeight = startHeight - chunks - chunksRemainder; - } else { - startHeight = (i + 1) * chunks; - stopHeight = nWorkers ? : startHeight - chunks; } auto result = std::make_shared(); From f06b7f1394ae2aa823ec5505332eed92ccd90d28 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 15 Feb 2023 08:32:45 +0000 Subject: [PATCH 7/7] Simplify further --- src/masternodes/rpc_accounts.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/masternodes/rpc_accounts.cpp b/src/masternodes/rpc_accounts.cpp index 4c831bfb5e..833a1c58d0 100644 --- a/src/masternodes/rpc_accounts.cpp +++ b/src/masternodes/rpc_accounts.cpp @@ -2016,8 +2016,8 @@ UniValue getburninfo(const JSONRPCRequest& request) { uint32_t stopHeight = startHeight - chunks; if (i + 1 == nWorkers) { - startHeight = (i + 1) * chunks + chunksRemainder; - stopHeight = startHeight - chunks - chunksRemainder; + startHeight += chunksRemainder; + stopHeight -= chunksRemainder; } auto result = std::make_shared();