From 9a0e59c09a164aa96c82d35ac4d44466952419e8 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Mon, 19 Apr 2021 10:38:40 +0300 Subject: [PATCH] Lazy rewards redistribution (#279) * Split custom tx processing into parsing and applying Signed-off-by: Anthony Fieroni * Prevent copies in serialize Signed-off-by: Anthony Fieroni * Adopt pool tables to have historical records Per account rewards calculation Signed-off-by: Anthony Fieroni * Do not redistribute rewards per block Signed-off-by: Anthony Fieroni * Fix lint circular dependency Signed-off-by: Anthony Fieroni * Introduce db version check at runtime Signed-off-by: Anthony Fieroni * Use bidirectional storage iterators Signed-off-by: Anthony Fieroni * Fix pool id serialization in update poolpair tx Signed-off-by: Anthony Fieroni * Update rewards on every block as drop-in replacement Signed-off-by: Anthony Fieroni * Split pool historical records * Added EunosHeight * Fork out rewards redistribution per block Signed-off-by: Anthony Fieroni * Extend pool swap functional test to Eunos hieght Signed-off-by: Anthony Fieroni * Minimize undo data size Signed-off-by: Anthony Fieroni * Prune undo data prior last checkpoint Signed-off-by: Anthony Fieroni * Prevent database fragmentation Signed-off-by: Anthony Fieroni * Make account history own database (#323) Signed-off-by: Anthony Fieroni Co-authored-by: surangap Co-authored-by: Ahmed Hilali --- src/Makefile.am | 4 +- src/amount.h | 6 +- src/chainparams.cpp | 11 + src/consensus/params.h | 4 + src/consensus/tx_check.cpp | 67 +- src/consensus/tx_check.h | 5 +- src/consensus/tx_verify.cpp | 3 +- src/dbwrapper.cpp | 1 + src/dbwrapper.h | 1 + src/flushablestorage.h | 258 ++- src/init.cpp | 22 +- src/masternodes/accounts.cpp | 22 +- src/masternodes/accounts.h | 7 +- src/masternodes/accountshistory.cpp | 114 +- src/masternodes/accountshistory.h | 83 +- src/masternodes/balances.h | 31 +- .../govvariables/lp_daily_dfi_reward.cpp | 6 +- .../govvariables/lp_daily_dfi_reward.h | 3 +- src/masternodes/govvariables/lp_splits.cpp | 16 +- src/masternodes/govvariables/lp_splits.h | 19 +- src/masternodes/gv.h | 7 +- src/masternodes/incentivefunding.cpp | 2 +- src/masternodes/incentivefunding.h | 2 +- src/masternodes/masternodes.cpp | 158 +- src/masternodes/masternodes.h | 43 +- src/masternodes/mn_checks.cpp | 1881 ++++++++--------- src/masternodes/mn_checks.h | 207 +- src/masternodes/mn_rpc.cpp | 43 +- src/masternodes/mn_rpc.h | 1 + src/masternodes/poolpairs.cpp | 557 ++++- src/masternodes/poolpairs.h | 279 +-- src/masternodes/res.h | 18 +- src/masternodes/rewardhistoryold.h | 38 - src/masternodes/rpc_accounts.cpp | 340 ++- src/masternodes/rpc_customtx.cpp | 198 ++ src/masternodes/rpc_masternodes.cpp | 20 +- src/masternodes/rpc_poolpair.cpp | 83 +- src/masternodes/rpc_tokens.cpp | 118 +- src/masternodes/tokens.cpp | 84 +- src/masternodes/tokens.h | 10 +- src/masternodes/undo.h | 2 +- src/masternodes/undos.cpp | 8 +- src/masternodes/undos.h | 8 +- src/miner.cpp | 5 +- src/rpc/blockchain.cpp | 1 + src/rpc/rawtransaction_util.cpp | 13 +- src/serialize.h | 8 + src/test/applytx_tests.cpp | 8 +- src/test/liquidity_tests.cpp | 209 +- src/test/storage_tests.cpp | 156 +- src/txmempool.cpp | 2 +- src/validation.cpp | 121 +- test/functional/feature_accounts_n_utxos.py | 3 +- test/functional/feature_autoauth.py | 4 +- test/functional/feature_poolpair.py | 2 +- test/functional/feature_poolswap_mechanism.py | 37 +- test/functional/feature_token_fork.py | 4 +- test/functional/feature_tokens_minting.py | 2 +- test/functional/feature_tokens_multisig.py | 2 +- test/functional/rpc_blockchain.py | 1 + test/functional/rpc_getcustomtx.py | 2 +- test/functional/rpc_listaccounthistory.py | 31 +- test/lint/lint-circular-dependencies.sh | 2 +- 63 files changed, 2917 insertions(+), 2486 deletions(-) delete mode 100644 src/masternodes/rewardhistoryold.h create mode 100644 src/masternodes/rpc_customtx.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 3dedc3e176d..492097361b8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -166,7 +166,6 @@ DEFI_CORE_H = \ masternodes/mn_checks.h \ masternodes/mn_rpc.h \ masternodes/res.h \ - masternodes/rewardhistoryold.h \ masternodes/tokens.h \ masternodes/poolpairs.h \ masternodes/undo.h \ @@ -374,8 +373,9 @@ libdefi_server_a_SOURCES = \ masternodes/masternodes.cpp \ masternodes/mn_checks.cpp \ masternodes/mn_rpc.cpp \ - masternodes/rpc_masternodes.cpp \ masternodes/rpc_accounts.cpp \ + masternodes/rpc_customtx.cpp \ + masternodes/rpc_masternodes.cpp \ masternodes/rpc_tokens.cpp \ masternodes/rpc_poolpair.cpp \ masternodes/tokens.cpp \ diff --git a/src/amount.h b/src/amount.h index 79a201ad939..3e8695e4fb0 100644 --- a/src/amount.h +++ b/src/amount.h @@ -70,7 +70,7 @@ struct DCT_ID { template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(v); + READWRITE(VARINT(v)); } }; @@ -125,7 +125,7 @@ struct CTokenAmount { // simple std::pair is less informative // add auto sumRes = SafeAdd(this->nValue, amount); if (!sumRes.ok) { - return sumRes.res(); + return std::move(sumRes); } this->nValue = *sumRes.val; return Res::Ok(); @@ -162,7 +162,7 @@ struct CTokenAmount { // simple std::pair is less informative template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nTokenId.v)); + READWRITE(nTokenId); READWRITE(nValue); } diff --git a/src/chainparams.cpp b/src/chainparams.cpp index bdaab4e682b..d954dad5054 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -813,6 +813,17 @@ void CRegTestParams::UpdateActivationParametersFromArgs(const ArgsManager& args) consensus.DakotaCrescentHeight = static_cast(height); } + if (gArgs.IsArgSet("-eunosheight")) { + int64_t height = gArgs.GetArg("-eunosheight", consensus.EunosHeight); + if (height < -1 || height >= std::numeric_limits::max()) { + throw std::runtime_error(strprintf("Activation height %ld for Eunos is out of valid range. Use -1 to disable Eunos features.", height)); + } else if (height == -1) { + LogPrintf("Eunos disabled for testing\n"); + height = std::numeric_limits::max(); + } + consensus.EunosHeight = static_cast(height); + } + if (!args.IsArgSet("-vbparams")) return; for (const std::string& strDeployment : args.GetArgs("-vbparams")) { diff --git a/src/consensus/params.h b/src/consensus/params.h index 6b12005096e..52c2e27ca33 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -105,6 +105,10 @@ struct Params { }; PoS pos; + uint32_t blocksPerDay() const { + static const uint32_t blocks = 60 * 60 * 24 / pos.nTargetSpacing; + return blocks; + } /** * Minimum blocks including miner confirmation of the total of 2016 blocks in a retargeting period, * (nTargetTimespan / nTargetSpacing) which is also used for BIP9 deployments. diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index ac139635841..9edfc0a8dcf 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -65,50 +65,39 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fChe return true; } -bool IsCriminalProofTx(CTransaction const & tx, std::vector & metadata) +bool ParseScriptByMarker(CScript const & script, + const std::vector & marker, + std::vector & metadata) { - if (!tx.IsCoinBase() || tx.vout.size() != 1 || tx.vout[0].nValue != 0) { - return false; - } - CScript const & memo = tx.vout[0].scriptPubKey; - CScript::const_iterator pc = memo.begin(); opcodetype opcode; - if (!memo.GetOp(pc, opcode) || opcode != OP_RETURN) { + auto pc = script.begin(); + if (!script.GetOp(pc, opcode) || opcode != OP_RETURN) { return false; } - if (!memo.GetOp(pc, opcode, metadata) || - (opcode > OP_PUSHDATA1 && - opcode != OP_PUSHDATA2 && - opcode != OP_PUSHDATA4) || - metadata.size() < DfCriminalTxMarker.size() + 1 || - memcmp(&metadata[0], &DfCriminalTxMarker[0], DfCriminalTxMarker.size()) != 0) { + if (!script.GetOp(pc, opcode, metadata) + || (opcode > OP_PUSHDATA1 && opcode != OP_PUSHDATA2 && opcode != OP_PUSHDATA4) + || metadata.size() < marker.size() + 1 + || memcmp(&metadata[0], &marker[0], marker.size()) != 0) { return false; } - metadata.erase(metadata.begin(), metadata.begin() + DfCriminalTxMarker.size()); + metadata.erase(metadata.begin(), metadata.begin() + marker.size()); return true; } -bool IsAnchorRewardTx(CTransaction const & tx, std::vector & metadata) +bool IsCriminalProofTx(CTransaction const & tx, std::vector & metadata) { - if (!tx.IsCoinBase() || tx.vout.size() != 2 || tx.vout[0].nValue != 0) { - return false; - } - CScript const & memo = tx.vout[0].scriptPubKey; - CScript::const_iterator pc = memo.begin(); - opcodetype opcode; - if (!memo.GetOp(pc, opcode) || opcode != OP_RETURN) { + if (!tx.IsCoinBase() || tx.vout.size() != 1 || tx.vout[0].nValue != 0) { return false; } - if (!memo.GetOp(pc, opcode, metadata) || - (opcode > OP_PUSHDATA1 && - opcode != OP_PUSHDATA2 && - opcode != OP_PUSHDATA4) || - metadata.size() < DfAnchorFinalizeTxMarker.size() + 1 || - memcmp(&metadata[0], &DfAnchorFinalizeTxMarker[0], DfAnchorFinalizeTxMarker.size()) != 0) { + return ParseScriptByMarker(tx.vout[0].scriptPubKey, DfCriminalTxMarker, metadata); +} + +bool IsAnchorRewardTx(CTransaction const & tx, std::vector & metadata) +{ + if (!tx.IsCoinBase() || tx.vout.size() != 2 || tx.vout[0].nValue != 0) { return false; } - metadata.erase(metadata.begin(), metadata.begin() + DfAnchorFinalizeTxMarker.size()); - return true; + return ParseScriptByMarker(tx.vout[0].scriptPubKey, DfAnchorFinalizeTxMarker, metadata); } bool IsAnchorRewardTxPlus(CTransaction const & tx, std::vector & metadata) @@ -116,21 +105,5 @@ bool IsAnchorRewardTxPlus(CTransaction const & tx, std::vector & if (!tx.IsCoinBase() || tx.vout.size() != 2 || tx.vout[0].nValue != 0) { return false; } - CScript const & memo = tx.vout[0].scriptPubKey; - CScript::const_iterator pc = memo.begin(); - opcodetype opcode; - if (!memo.GetOp(pc, opcode) || opcode != OP_RETURN) { - return false; - } - if (!memo.GetOp(pc, opcode, metadata) || - (opcode > OP_PUSHDATA1 && - opcode != OP_PUSHDATA2 && - opcode != OP_PUSHDATA4) || - metadata.size() < DfAnchorFinalizeTxMarkerPlus.size() + 1 || - memcmp(&metadata[0], &DfAnchorFinalizeTxMarkerPlus[0], DfAnchorFinalizeTxMarkerPlus.size()) != 0) { - return false; - } - metadata.erase(metadata.begin(), metadata.begin() + DfAnchorFinalizeTxMarkerPlus.size()); - return true; + return ParseScriptByMarker(tx.vout[0].scriptPubKey, DfAnchorFinalizeTxMarkerPlus, metadata); } - diff --git a/src/consensus/tx_check.h b/src/consensus/tx_check.h index aa621ca4371..d92aebe4ec5 100644 --- a/src/consensus/tx_check.h +++ b/src/consensus/tx_check.h @@ -19,12 +19,15 @@ extern const std::vector DfCriminalTxMarker; extern const std::vector DfAnchorFinalizeTxMarker; extern const std::vector DfAnchorFinalizeTxMarkerPlus; +class CScript; class CTransaction; class CValidationState; bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); -/// moved here (!!) due to strange linker errors under mac/win builds +bool ParseScriptByMarker(CScript const & script, + const std::vector & marker, + std::vector & metadata); bool IsCriminalProofTx(CTransaction const & tx, std::vector & metadata); bool IsAnchorRewardTx(CTransaction const & tx, std::vector & metadata); bool IsAnchorRewardTxPlus(CTransaction const & tx, std::vector & metadata); diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index ba2329fc0bd..646e167567f 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -221,7 +221,8 @@ bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, c const auto txType = GuessCustomTxType(tx, dummy); if (NotAllowedToFail(txType, nSpendHeight)) { - auto res = ApplyCustomTx(const_cast(*mnview), inputs, tx, chainparams.GetConsensus(), nSpendHeight, 0, uint64_t{0}, true); // note for 'isCheck == true' here; 'zero' for txn is dummy value + CCustomCSView discardCache(const_cast(*mnview)); + auto res = ApplyCustomTx(discardCache, inputs, tx, chainparams.GetConsensus(), nSpendHeight); if (!res.ok && (res.code & CustomTxErrCodes::Fatal)) { return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-customtx", res.msg); } diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 61602b5bc15..54e6263cf59 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -237,6 +237,7 @@ CDBIterator::~CDBIterator() { delete piter; } bool CDBIterator::Valid() const { return piter->Valid(); } void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } void CDBIterator::Next() { piter->Next(); } +void CDBIterator::Prev() { piter->Prev(); } namespace dbwrapper_private { diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 06b2ce67360..b0cb975ea2c 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -159,6 +159,7 @@ class CDBIterator } void Next(); + void Prev(); template bool GetKey(K& key) { leveldb::Slice slKey = piter->key(); diff --git a/src/flushablestorage.h b/src/flushablestorage.h index cd0484a846b..d017b8cb5ee 100644 --- a/src/flushablestorage.h +++ b/src/flushablestorage.h @@ -18,15 +18,16 @@ using MapKV = std::map>; template static TBytes DbTypeToBytes(const T& value) { - CDataStream stream(SER_DISK, CLIENT_VERSION); + TBytes bytes; + CVectorWriter stream(SER_DISK, CLIENT_VERSION, bytes, 0); stream << value; - return TBytes(stream.begin(), stream.end()); + return bytes; } template static bool BytesToDbType(const TBytes& bytes, T& value) { try { - CDataStream stream(bytes, SER_DISK, CLIENT_VERSION); + VectorReader stream(SER_DISK, CLIENT_VERSION, bytes, 0); stream >> value; // assert(stream.size() == 0); // will fail with partial key matching } @@ -42,6 +43,7 @@ class CStorageKVIterator { virtual ~CStorageKVIterator() = default; virtual void Seek(const TBytes& key) = 0; virtual void Next() = 0; + virtual void Prev() = 0; virtual bool Valid() = 0; virtual TBytes Key() = 0; virtual TBytes Value() = 0; @@ -57,6 +59,7 @@ class CStorageKV { virtual bool Read(const TBytes& key, TBytes& value) const = 0; virtual std::unique_ptr NewIterator() = 0; virtual size_t SizeEstimate() const = 0; + virtual void Discard() = 0; virtual bool Flush() = 0; }; @@ -66,15 +69,13 @@ struct RawTBytes { std::reference_wrapper ref; template - void Serialize(Stream& os) const - { + void Serialize(Stream& os) const { auto& val = ref.get(); os.write((char*)val.data(), val.size()); } template - void Unserialize(Stream& is) - { + void Unserialize(Stream& is) { auto& val = ref.get(); val.resize(is.size()); is.read((char*)val.data(), is.size()); @@ -82,8 +83,7 @@ struct RawTBytes { }; template -inline RawTBytes refTBytes(T& val) -{ +inline RawTBytes refTBytes(T& val) { return RawTBytes{val}; } @@ -97,7 +97,12 @@ class CStorageLevelDBIterator : public CStorageKVIterator { void Seek(const TBytes& key) override { it->Seek(refTBytes(key)); // lower_bound in fact } - void Next() override { it->Next(); } + void Next() override { + it->Next(); + } + void Prev() override { + it->Prev(); + } bool Valid() override { return it->Valid(); } @@ -130,6 +135,7 @@ class CStorageLevelDB : public CStorageKV { return true; } bool Erase(const TBytes& key) override { + begin.empty() ? (begin = key) : (end = key); batch.Erase(refTBytes(key)); return true; } @@ -140,16 +146,32 @@ class CStorageLevelDB : public CStorageKV { bool Flush() override { // Commit batch auto result = db.WriteBatch(batch); batch.Clear(); + // prevent db fragmentation + if (!begin.empty() && !end.empty()) { + db.CompactRange(refTBytes(begin), refTBytes(end)); + } + end.clear(); + begin.clear(); return result; } + void Discard() override { + end.clear(); + begin.clear(); + batch.Clear(); + } size_t SizeEstimate() const override { return batch.SizeEstimate(); } std::unique_ptr NewIterator() override { return MakeUnique(std::unique_ptr(db.NewIterator())); } + bool IsEmpty() { + return db.IsEmpty(); + } private: + TBytes end; + TBytes begin; CDBWrapper db; CDBBatch batch; }; @@ -159,80 +181,83 @@ class CStorageLevelDB : public CStorageKV { // Flushable Key-Value Storage Iterator class CFlushableStorageKVIterator : public CStorageKVIterator { public: - explicit CFlushableStorageKVIterator(std::unique_ptr&& pIt_, MapKV& map_) : pIt{std::move(pIt_)}, map(map_) { - inited = parentOk = mapOk = useMap = false; + explicit CFlushableStorageKVIterator(std::unique_ptr&& pIt, MapKV& map) : map(map), pIt(std::move(pIt)) { + itState = Invalid; } CFlushableStorageKVIterator(const CFlushableStorageKVIterator&) = delete; ~CFlushableStorageKVIterator() override = default; void Seek(const TBytes& key) override { - prevKey.clear(); pIt->Seek(key); - parentOk = pIt->Valid(); - mIt = map.lower_bound(key); - mapOk = mIt != map.end(); - inited = true; - Next(); + mIt = Advance(map.lower_bound(key), map.end(), std::greater{}, {}); } void Next() override { - if (!inited) throw std::runtime_error("Iterator wasn't inited."); - - if (!prevKey.empty()) { - useMap ? nextMap() : nextParent(); + assert(Valid()); + mIt = Advance(mIt, map.end(), std::greater{}, Key()); + } + void Prev() override { + assert(Valid()); + auto tmp = mIt; + if (tmp != map.end()) { + ++tmp; } - - while (mapOk || parentOk) { - if (mapOk) { - while (mapOk && (!parentOk || mIt->first <= pIt->Key())) { - bool ok = false; - - if (mIt->second) { - ok = prevKey.empty() || mIt->first > prevKey; - } else { - prevKey = mIt->first; - } - if (ok) { - useMap = true; - prevKey = mIt->first; - return; - } - nextMap(); - } - } - if (parentOk) { - if (prevKey.empty() || pIt->Key() > prevKey) { - useMap = false; - prevKey = pIt->Key(); - return; - } - nextParent(); - } + auto it = std::reverse_iterator(tmp); + auto end = Advance(it, map.rend(), std::less{}, Key()); + if (end == map.rend()) { + mIt = map.begin(); + } else { + auto offset = mIt == map.end() ? 1 : 0; + std::advance(mIt, -std::distance(it, end) - offset); } } bool Valid() override { - return mapOk || parentOk; + return itState != Invalid; } TBytes Key() override { - return useMap ? mIt->first : pIt->Key(); + assert(Valid()); + return itState == Map ? mIt->first : pIt->Key(); } TBytes Value() override { - return useMap ? *mIt->second : pIt->Value(); + assert(Valid()); + return itState == Map ? *mIt->second : pIt->Value(); } private: - void nextMap() { - mapOk = mapOk && ++mIt != map.end(); + template + TIterator Advance(TIterator it, TIterator end, Compare comp, TBytes prevKey) { + + while (it != end || pIt->Valid()) { + while (it != end && (!pIt->Valid() || !comp(it->first, pIt->Key()))) { + if (prevKey.empty() || comp(it->first, prevKey)) { + if (it->second) { + itState = Map; + return it; + } else { + prevKey = it->first; + } + } + ++it; + } + if (pIt->Valid()) { + if (prevKey.empty() || comp(pIt->Key(), prevKey)) { + itState = Parent; + return it; + } + NextParent(it); + } + } + itState = Invalid; + return it; } - void nextParent() { - parentOk = parentOk && (pIt->Next(), pIt->Valid()); + void NextParent(MapKV::const_iterator&) { + pIt->Next(); + } + void NextParent(std::reverse_iterator&) { + pIt->Prev(); } - bool inited; - bool useMap; - std::unique_ptr pIt; - bool parentOk; const MapKV& map; MapKV::const_iterator mIt; - bool mapOk; - TBytes prevKey; + std::unique_ptr pIt; + enum IteratorState { Invalid, Map, Parent } itState; }; // Flushable Key-Value Storage @@ -281,6 +306,9 @@ class CFlushableStorageKV : public CStorageKV { changed.clear(); return true; } + void Discard() override { + changed.clear(); + } size_t SizeEstimate() const override { return memusage::DynamicUsage(changed); } @@ -298,30 +326,95 @@ class CFlushableStorageKV : public CStorageKV { }; template -class CLazySerialize -{ +class CLazySerialize { Optional value; - CStorageKVIterator& it; + std::unique_ptr& it; public: CLazySerialize(const CLazySerialize&) = default; - explicit CLazySerialize(CStorageKVIterator& it) : it(it) {} + explicit CLazySerialize(std::unique_ptr& it) : it(it) {} - operator T() - { + operator T() & { return get(); } - - const T& get() - { + operator T() && { + get(); + return std::move(*value); + } + const T& get() { if (!value) { value = T{}; - BytesToDbType(it.Value(), *value); + BytesToDbType(it->Value(), *value); } return *value; } }; +template +class CStorageIteratorWrapper { + bool valid = false; + std::pair key; + std::unique_ptr it; + + void UpdateValidity() { + valid = it->Valid() && BytesToDbType(it->Key(), key) && key.first == By::prefix; + } + + struct Resolver { + std::unique_ptr& it; + + template + inline operator CLazySerialize() { + return CLazySerialize{it}; + } + template + inline T as() { + return CLazySerialize{it}; + } + template + inline operator T() { + return as(); + } + }; + +public: + CStorageIteratorWrapper(CStorageIteratorWrapper&&) = default; + CStorageIteratorWrapper(std::unique_ptr it) : it(std::move(it)) {} + + CStorageIteratorWrapper& operator=(CStorageIteratorWrapper&& other) { + valid = other.valid; + it = std::move(other.it); + key = std::move(other.key); + return *this; + } + bool Valid() { + return valid; + } + const KeyType& Key() { + assert(Valid()); + return key.second; + } + Resolver Value() { + assert(Valid()); + return Resolver{it}; + } + void Next() { + assert(Valid()); + it->Next(); + UpdateValidity(); + } + void Prev() { + assert(Valid()); + it->Prev(); + UpdateValidity(); + } + void Seek(const KeyType& newKey) { + key = std::make_pair(By::prefix, newKey); + it->Seek(DbTypeToBytes(key)); + UpdateValidity(); + } +}; + class CStorageView { public: CStorageView() = default; @@ -376,24 +469,25 @@ class CStorageView { return {result}; return {}; } - + template + CStorageIteratorWrapper LowerBound(KeyType const & key) { + CStorageIteratorWrapper it{DB().NewIterator()}; + it.Seek(key); + return it; + } template - bool ForEach(std::function)> callback, KeyType const & start = KeyType()) const { - auto& self = const_cast(*this); - auto key = std::make_pair(By::prefix, start); - - auto it = self.DB().NewIterator(); - for(it->Seek(DbTypeToBytes(key)); it->Valid() && BytesToDbType(it->Key(), key) && key.first == By::prefix; it->Next()) { + void ForEach(std::function)> callback, KeyType const & start = {}) { + for(auto it = LowerBound(start); it.Valid(); it.Next()) { boost::this_thread::interruption_point(); - if (!callback(key.second, CLazySerialize(*it))) + if (!callback(it.Key(), it.Value())) { break; + } } - return true; } bool Flush() { return DB().Flush(); } - + void Discard() { DB().Discard(); } size_t SizeEstimate() const { return DB().SizeEstimate(); } protected: diff --git a/src/init.cpp b/src/init.cpp index 4182beba941..2cce7b7d2ea 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -418,7 +418,6 @@ void SetupServerArgs() #endif gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-acindex", strprintf("Maintain a full account history index, tracking all accounts balances changes. Used by the listaccounthistory and accounthistorycount rpc calls (default: %u)", false), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-acindex-mineonly", strprintf("Maintain a mine only account history index, tracking mine accounts balances changes. Used by the listaccounthistory and accounthistorycount rpc calls (default: %u)", false), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-blockfilterindex=", strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) + " If is not supplied or if = 1, indexes for all known types are enabled.", @@ -469,6 +468,7 @@ void SetupServerArgs() gArgs.AddArg("-clarkequayheight", "ClarkeQuay fork activation height (regtest only)", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); gArgs.AddArg("-dakotaheight", "Dakota fork activation height (regtest only)", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); gArgs.AddArg("-dakotacrescentheight", "DakotaCrescent fork activation height (regtest only)", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); + gArgs.AddArg("-eunosheight", "Eunos fork activation height (regtest only)", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); #ifdef USE_UPNP #if USE_UPNP gArgs.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1506,11 +1506,6 @@ bool AppInitMain(InitInterfaces& interfaces) LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); - if (gArgs.GetBoolArg("-acindex", DEFAULT_ACINDEX) && gArgs.GetBoolArg("-acindex-mineonly", DEFAULT_ACINDEX_MINEONLY)) { - LogPrintf("Warning -acindex and -acindex-mineonly are set, -acindex will be used\n"); - gArgs.ForceSetArg("-acindex-mineonly", "0"); - } - bool fLoaded = false; while (!fLoaded && !ShutdownRequested()) { bool fReset = fReindex; @@ -1595,13 +1590,22 @@ bool AppInitMain(InitInterfaces& interfaces) pcustomcsDB = MakeUnique(GetDataDir() / "enhancedcs", nCustomCacheSize, false, fReset || fReindexChainState); pcustomcsview.reset(); pcustomcsview = MakeUnique(*pcustomcsDB.get()); - if (!fReset && (gArgs.GetBoolArg("-acindex", DEFAULT_ACINDEX) || gArgs.GetBoolArg("-acindex-mineonly", DEFAULT_ACINDEX_MINEONLY))) { - if (shouldMigrateOldRewardHistory(*pcustomcsview)) { - strLoadError = _("Account history needs rebuild").translated; + if (!fReset && !fReindexChainState) { + if (!pcustomcsDB->IsEmpty() && pcustomcsview->GetDbVersion() != CCustomCSView::DbVersion) { + strLoadError = _("Account database is unsuitable").translated; break; } } + // Ensure we are on latest DB version + pcustomcsview->SetDbVersion(CCustomCSView::DbVersion); + + // make account history db + paccountHistoryDB.reset(); + if (gArgs.GetBoolArg("-acindex", DEFAULT_ACINDEX)) { + paccountHistoryDB = MakeUnique(GetDataDir() / "history", nCustomCacheSize, false, fReset || fReindexChainState); + } + // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!::ChainstateActive().CoinsDB().Upgrade()) { diff --git a/src/masternodes/accounts.cpp b/src/masternodes/accounts.cpp index 3e95f470a8b..1be589debd6 100644 --- a/src/masternodes/accounts.cpp +++ b/src/masternodes/accounts.cpp @@ -6,8 +6,9 @@ /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!! const unsigned char CAccountsView::ByBalanceKey::prefix = 'a'; +const unsigned char CAccountsView::ByHeightKey::prefix = 'b'; -void CAccountsView::ForEachBalance(std::function callback, BalanceKey const & start) const +void CAccountsView::ForEachBalance(std::function callback, BalanceKey const & start) { ForEach([&callback] (BalanceKey const & key, CAmount val) { return callback(key.owner, CTokenAmount{key.tokenID, val}); @@ -82,3 +83,22 @@ Res CAccountsView::SubBalances(CScript const & owner, CBalances const & balances return Res::Ok(); } +void CAccountsView::ForEachAccount(std::function callback) +{ + ForEach([&callback] (CScript const & owner, CLazySerialize) { + return callback(owner); + }); +} + +Res CAccountsView::UpdateBalancesHeight(CScript const & owner, uint32_t height) +{ + WriteBy(owner, height); + return Res::Ok(); +} + +uint32_t CAccountsView::GetBalancesHeight(CScript const & owner) +{ + uint32_t height; + bool ok = ReadBy(owner, height); + return ok ? height : 0; +} diff --git a/src/masternodes/accounts.h b/src/masternodes/accounts.h index 8e216644e50..d04c62dbeea 100644 --- a/src/masternodes/accounts.h +++ b/src/masternodes/accounts.h @@ -14,7 +14,8 @@ class CAccountsView : public virtual CStorageView { public: - void ForEachBalance(std::function callback, BalanceKey const & start = {}) const; + void ForEachAccount(std::function callback); + void ForEachBalance(std::function callback, BalanceKey const & start = {}); CTokenAmount GetBalance(CScript const & owner, DCT_ID tokenID) const; virtual Res AddBalance(CScript const & owner, CTokenAmount amount); @@ -23,8 +24,12 @@ class CAccountsView : public virtual CStorageView Res AddBalances(CScript const & owner, CBalances const & balances); Res SubBalances(CScript const & owner, CBalances const & balances); + uint32_t GetBalancesHeight(CScript const & owner); + Res UpdateBalancesHeight(CScript const & owner, uint32_t height); + // tags struct ByBalanceKey { static const unsigned char prefix; }; + struct ByHeightKey { static const unsigned char prefix; }; private: Res SetBalance(CScript const & owner, CTokenAmount amount); diff --git a/src/masternodes/accountshistory.cpp b/src/masternodes/accountshistory.cpp index 10dffb4e834..b56b3b3744a 100644 --- a/src/masternodes/accountshistory.cpp +++ b/src/masternodes/accountshistory.cpp @@ -5,93 +5,95 @@ #include #include #include -#include #include -#include +/// @Note it's in own database +const unsigned char CAccountsHistoryView::ByAccountHistoryKey::prefix = 'h'; -/// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!! -const unsigned char CAccountsHistoryView::ByMineAccountHistoryKey::prefix = 'm'; -const unsigned char CAccountsHistoryView::ByAllAccountHistoryKey::prefix = 'h'; -const unsigned char CRewardsHistoryView::ByMineRewardHistoryKey::prefix = 'Q'; -const unsigned char CRewardsHistoryView::ByAllRewardHistoryKey::prefix = 'W'; +void CAccountsHistoryView::ForEachAccountHistory(std::function)> callback, AccountHistoryKey const & start) +{ + ForEach(callback, start); +} -void CAccountsHistoryView::ForEachMineAccountHistory(std::function)> callback, AccountHistoryKey const & start) const +Res CAccountsHistoryView::WriteAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value) { - ForEach(callback, start); + WriteBy(key, value); + return Res::Ok(); } -Res CAccountsHistoryView::SetMineAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value) +Res CAccountsHistoryView::EraseAccountHistory(const AccountHistoryKey& key) { - WriteBy(key, value); + EraseBy(key); return Res::Ok(); } -void CAccountsHistoryView::ForEachAllAccountHistory(std::function)> callback, AccountHistoryKey const & start) const +CAccountHistoryStorage::CAccountHistoryStorage(const fs::path& dbName, std::size_t cacheSize, bool fMemory, bool fWipe) + : CStorageView(new CStorageLevelDB(dbName, cacheSize, fMemory, fWipe)) { - ForEach(callback, start); } -Res CAccountsHistoryView::SetAllAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value) +CAccountsHistoryWriter::CAccountsHistoryWriter(CCustomCSView & storage, uint32_t height, uint32_t txn, const uint256& txid, uint8_t type, CAccountsHistoryView* historyView) + : CStorageView(new CFlushableStorageKV(storage.GetRaw())), height(height), txn(txn), txid(txid), type(type), historyView(historyView) { - WriteBy(key, value); - return Res::Ok(); } -void CRewardsHistoryView::ForEachMineRewardHistory(std::function)> callback, RewardHistoryKey const & start) const +Res CAccountsHistoryWriter::AddBalance(CScript const & owner, CTokenAmount amount) { - ForEach(callback, start); + auto res = CCustomCSView::AddBalance(owner, amount); + if (historyView && res.ok && amount.nValue != 0) { + diffs[owner][amount.nTokenId] += amount.nValue; + } + return res; } -Res CRewardsHistoryView::SetMineRewardHistory(const RewardHistoryKey& key, const RewardHistoryValue& value) +Res CAccountsHistoryWriter::SubBalance(CScript const & owner, CTokenAmount amount) { - WriteBy(key, value); - return Res::Ok(); + auto res = CCustomCSView::SubBalance(owner, amount); + if (historyView && res.ok && amount.nValue != 0) { + diffs[owner][amount.nTokenId] -= amount.nValue; + } + return res; } -void CRewardsHistoryView::ForEachAllRewardHistory(std::function)> callback, RewardHistoryKey const & start) const +bool CAccountsHistoryWriter::Flush() { - ForEach(callback, start); + if (historyView) { + for (const auto& diff : diffs) { + historyView->WriteAccountHistory({diff.first, height, txn}, {txid, type, diff.second}); + } + } + return CCustomCSView::Flush(); } -Res CRewardsHistoryView::SetAllRewardHistory(const RewardHistoryKey& key, const RewardHistoryValue& value) +CAccountsHistoryEraser::CAccountsHistoryEraser(CCustomCSView & storage, uint32_t height, uint32_t txn, CAccountsHistoryView* historyView) + : CStorageView(new CFlushableStorageKV(storage.GetRaw())), height(height), txn(txn), historyView(historyView) { - WriteBy(key, value); +} + +Res CAccountsHistoryEraser::AddBalance(CScript const & owner, CTokenAmount) +{ + if (historyView) { + accounts.insert(owner); + } return Res::Ok(); } -bool shouldMigrateOldRewardHistory(CCustomCSView & view) +Res CAccountsHistoryEraser::SubBalance(CScript const & owner, CTokenAmount) { - auto it = view.GetRaw().NewIterator(); - try { - auto prefix = oldRewardHistoryPrefix; - auto oldKey = std::make_pair(prefix, oldRewardHistoryKey{}); - it->Seek(DbTypeToBytes(oldKey)); - if (it->Valid() && BytesToDbType(it->Key(), oldKey) && oldKey.first == prefix) { - return true; - } - prefix = CRewardsHistoryView::ByMineRewardHistoryKey::prefix; - auto newKey = std::make_pair(prefix, RewardHistoryKey{}); - it->Seek(DbTypeToBytes(newKey)); - if (it->Valid() && BytesToDbType(it->Key(), newKey) && newKey.first == prefix) { - return false; - } - prefix = CRewardsHistoryView::ByAllRewardHistoryKey::prefix; - newKey = std::make_pair(prefix, RewardHistoryKey{}); - it->Seek(DbTypeToBytes(newKey)); - if (it->Valid() && BytesToDbType(it->Key(), newKey) && newKey.first == prefix) { - return false; + if (historyView) { + accounts.insert(owner); + } + return Res::Ok(); +} + +bool CAccountsHistoryEraser::Flush() +{ + if (historyView) { + for (const auto& account : accounts) { + historyView->EraseAccountHistory({account, height, txn}); } - bool hasOldAccountHistory = false; - view.ForEachAllAccountHistory([&](AccountHistoryKey const & key, CLazySerialize) { - if (key.txn == std::numeric_limits::max()) { - hasOldAccountHistory = true; - return false; - } - return true; - }, { {}, 0, std::numeric_limits::max() }); - return hasOldAccountHistory; - } catch(...) { - return true; } + return CCustomCSView::Flush(); } + +std::unique_ptr paccountHistoryDB; diff --git a/src/masternodes/accountshistory.h b/src/masternodes/accountshistory.h index b486f1df9a6..3e0066a20e3 100644 --- a/src/masternodes/accountshistory.h +++ b/src/masternodes/accountshistory.h @@ -7,10 +7,10 @@ #include #include +#include #include