Merge pull request #128 from bvbfan/feature/accounthistory

Better handling of account history

src/flushablestorage.h

@@ -313,10 +313,29 @@ class CFlushableStorageKV : public CStorageKV {
     MapKV changed;
 };
 
+template<typename T>
+struct CLazySerialize
+{
+    CStorageKVIterator& it;
+
+    operator T()
+    {
+        return get();
+    }
+
+    T get()
+    {
+        T value;
+        BytesToDbType(it.Value(), value);
+        return value;
+    }
+};
+
 class CStorageView {
 public:
+    CStorageView() = default;
     CStorageView(CStorageKV * st) : storage(st) {}
-    CStorageView() {}
+    virtual ~CStorageView() = default;
 
     template<typename KeyType>
     bool Exists(const KeyType& key) const {
@@ -374,18 +393,15 @@ class CStorageView {
     }
 
     template<typename By, typename KeyType, typename ValueType>
-    bool ForEach(std::function<bool(KeyType const &, ValueType &)> callback, KeyType const & start = KeyType()) const {
+    bool ForEach(std::function<bool(KeyType const &, CLazySerialize<ValueType>)> callback, KeyType const & start = KeyType()) const {
         auto& self = const_cast<CStorageView&>(*this);
         auto key = std::make_pair(By::prefix, start);
 
         auto it = self.DB().NewIterator();
         for(it->Seek(DbTypeToBytes(key)); it->Valid() && (BytesToDbType(it->Key(), key), key.first == By::prefix); it->Next()) {
             boost::this_thread::interruption_point();
 
-            ValueType value;
-            BytesToDbType(it->Value(), value);
-
-            if (!callback(key.second, value))
+            if (!callback(key.second, CLazySerialize<ValueType>{*it}))
                 break;
         }
         return true;

src/init.cpp

@@ -415,7 +415,7 @@ void SetupServerArgs()
     hidden_args.emplace_back("-sysperms");
 #endif
     gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
-    gArgs.AddArg("-acindex", strprintf("Maintain a full account history index, tracking all accounts balances changes. Used by the listaccounthistory rpc call (default: %u)", false), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-acindex", strprintf("Maintain a full account history index, tracking all accounts balances changes. Used by the listaccounthistory and accounthistorycount rpc calls (default: %u)", false), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-blockfilterindex=<type>",
                  strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) +
                  " If <type> is not supplied or if <type> = 1, indexes for all known types are enabled.",
@@ -1585,6 +1585,27 @@ bool AppInitMain(InitInterfaces& interfaces)
                 pcustomcsDB = MakeUnique<CStorageLevelDB>(GetDataDir() / "enhancedcs", nMinDbCache << 20, false, fReset || fReindexChainState);
                 pcustomcsview.reset();
                 pcustomcsview = MakeUnique<CCustomCSView>(*pcustomcsDB.get());
+                if (!fReset && gArgs.GetBoolArg("-acindex", false)) {
+                    bool hasRewardHistory = false;
+                    pcustomcsview->ForEachRewardHistory([&](RewardHistoryKey const &, CLazySerialize<RewardHistoryValue>) {
+                        hasRewardHistory = true;
+                        return false;
+                    });
+                    if (!hasRewardHistory) {
+                        bool hasOldAccountHistory = false;
+                        pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const & key, CLazySerialize<AccountHistoryValue>) {
+                            if (key.txn == std::numeric_limits<uint32_t>::max()) {
+                                hasOldAccountHistory = true;
+                                return false;
+                            }
+                            return true;
+                        }, { {}, 0, std::numeric_limits<uint32_t>::max() });
+                        if (hasOldAccountHistory) {
+                            strLoadError = _("Account history needs rebuild").translated;
+                            break;
+                        }
+                    }
+                }
 
                 panchorauths.reset();
                 panchorauths = MakeUnique<CAnchorAuthIndex>();

src/masternodes/accounts.cpp

@@ -7,9 +7,9 @@
 /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!!
 const unsigned char CAccountsView::ByBalanceKey::prefix = 'a';
 
-void CAccountsView::ForEachBalance(std::function<bool(CScript const & owner, CTokenAmount const & amount)> callback, BalanceKey start) const
+void CAccountsView::ForEachBalance(std::function<bool(CScript const &, CTokenAmount const &)> callback, BalanceKey const & start) const
 {
-    ForEach<ByBalanceKey, BalanceKey, CAmount>([&callback] (BalanceKey const & key, CAmount const & val) {
+    ForEach<ByBalanceKey, BalanceKey, CAmount>([&callback] (BalanceKey const & key, CAmount val) {
         return callback(key.owner, CTokenAmount{key.tokenID, val});
     }, start);
 }

src/masternodes/accounts.h

@@ -14,12 +14,13 @@
 class CAccountsView : public virtual CStorageView
 {
 public:
-    void ForEachBalance(std::function<bool(CScript const & owner, CTokenAmount const & amount)> callback, BalanceKey start = {}) const;
+    void ForEachBalance(std::function<bool(CScript const &, CTokenAmount const &)> callback, BalanceKey const & start = {}) const;
     CTokenAmount GetBalance(CScript const & owner, DCT_ID tokenID) const;
 
-    Res AddBalance(CScript const & owner, CTokenAmount amount);
+    virtual Res AddBalance(CScript const & owner, CTokenAmount amount);
+    virtual Res SubBalance(CScript const & owner, CTokenAmount amount);
+
     Res AddBalances(CScript const & owner, CBalances const & balances);
-    Res SubBalance(CScript const & owner, CTokenAmount amount);
     Res SubBalances(CScript const & owner, CBalances const & balances);
 
     // tags

src/masternodes/accountshistory.cpp

@@ -10,53 +10,26 @@
 
 /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!!
 const unsigned char CAccountsHistoryView::ByAccountHistoryKey::prefix = 'h'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB
+const unsigned char CRewardsHistoryView::ByRewardHistoryKey::prefix = 'R'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB
 
-void CAccountsHistoryView::ForEachAccountHistory(std::function<bool(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diffs)> callback, AccountHistoryKey start) const
+void CAccountsHistoryView::ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, CLazySerialize<AccountHistoryValue>)> callback, AccountHistoryKey const & start) const
 {
-    ForEach<ByAccountHistoryKey, AccountHistoryKey, AccountHistoryValue>([&callback] (AccountHistoryKey const & key, AccountHistoryValue const & val) {
-        return callback(key.owner,key.blockHeight, key.txn, val.txid, val.category, val.diff);
-    }, start);
+    ForEach<ByAccountHistoryKey, AccountHistoryKey, AccountHistoryValue>(callback, start);
 }
 
-Res CAccountsHistoryView::SetAccountHistory(const CScript & owner, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category, TAmounts const & diff)
+Res CAccountsHistoryView::SetAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value)
 {
-////  left for debug:
-//    std::string ownerStr;
-//    CTxDestination dest;
-//    if (!ExtractDestination(owner, dest)) {
-//        ownerStr = owner.GetHex();
-//    } else
-//        ownerStr = EncodeDestination(dest);
-//    LogPrintf("DEBUG: SetAccountHistory: owner: %s, ownerStr: %s, block: %i, txn: %d, txid: %s, diffs: %ld\n", owner.GetHex().c_str(), ownerStr.c_str(), height, txn, txid.ToString().c_str(), diff.size());
-
-    WriteBy<ByAccountHistoryKey>(AccountHistoryKey{owner, height, txn}, AccountHistoryValue{txid, category, diff});
+    WriteBy<ByAccountHistoryKey>(key, value);
     return Res::Ok();
 }
 
-bool CAccountsHistoryView::TrackAffectedAccounts(CStorageKV const & before, MapKV const & diff, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category) {
-    // txn set to max if called from CreateNewBlock to check account balances, do not track here.
-    if (!gArgs.GetBoolArg("-acindex", false) || txn == std::numeric_limits<uint32_t>::max())
-        return false;
-
-    std::map<CScript, TAmounts> balancesDiff;
-    using TKey = std::pair<unsigned char, BalanceKey>;
-
-    for (auto it = diff.lower_bound({CAccountsView::ByBalanceKey::prefix}); it != diff.end() && it->first.at(0) == CAccountsView::ByBalanceKey::prefix; ++it) {
-        CAmount oldAmount = 0, newAmount = 0;
+void CRewardsHistoryView::ForEachRewardHistory(std::function<bool(RewardHistoryKey const &, CLazySerialize<RewardHistoryValue>)> callback, RewardHistoryKey const & start) const
+{
+    ForEach<ByRewardHistoryKey, RewardHistoryKey, RewardHistoryValue>(callback, start);
+}
 
-        if (it->second) {
-            BytesToDbType(*it->second, newAmount);
-        }
-        TBytes beforeVal;
-        if (before.Read(it->first, beforeVal)) {
-            BytesToDbType(beforeVal, oldAmount);
-        }
-        TKey balanceKey;
-        BytesToDbType(it->first, balanceKey);
-        balancesDiff[balanceKey.second.owner][balanceKey.second.tokenID] = newAmount - oldAmount;
-    }
-    for (auto const & kv : balancesDiff) {
-        SetAccountHistory(kv.first, height, txn, txid, category, kv.second);
-    }
-    return true;
+Res CRewardsHistoryView::SetRewardHistory(const RewardHistoryKey& key, const RewardHistoryValue& value)
+{
+    WriteBy<ByRewardHistoryKey>(key, value);
+    return Res::Ok();
 }

src/masternodes/accountshistory.h

@@ -53,15 +53,61 @@ struct AccountHistoryValue {
     }
 };
 
+struct RewardHistoryKey {
+    CScript owner;
+    uint32_t blockHeight;
+    DCT_ID poolID; // for order in block
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        READWRITE(owner);
+
+        if (ser_action.ForRead()) {
+            READWRITE(WrapBigEndian(blockHeight));
+            blockHeight = ~blockHeight;
+        }
+        else {
+            uint32_t blockHeight_ = ~blockHeight;
+            READWRITE(WrapBigEndian(blockHeight_));
+        }
+
+        READWRITE(VARINT(poolID.v));
+    }
+};
+
+struct RewardHistoryValue {
+    unsigned char category;
+    TAmounts diff;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        READWRITE(category);
+        READWRITE(diff);
+    }
+};
+
 class CAccountsHistoryView : public virtual CStorageView
 {
 public:
-    Res SetAccountHistory(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diff);
-    void ForEachAccountHistory(std::function<bool(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diff)> callback, AccountHistoryKey start) const;
-    bool TrackAffectedAccounts(CStorageKV const & before, MapKV const & diff, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category);
+    Res SetAccountHistory(AccountHistoryKey const & key, AccountHistoryValue const & value);
+    void ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, CLazySerialize<AccountHistoryValue>)> callback, AccountHistoryKey const & start = {}) const;
 
     // tags
     struct ByAccountHistoryKey { static const unsigned char prefix; };
 };
 
+class CRewardsHistoryView : public virtual CStorageView
+{
+public:
+    Res SetRewardHistory(RewardHistoryKey const & key, RewardHistoryValue const & value);
+    void ForEachRewardHistory(std::function<bool(RewardHistoryKey const &, CLazySerialize<RewardHistoryValue>)> callback, RewardHistoryKey const & start = {}) const;
+
+    // tags
+    struct ByRewardHistoryKey { static const unsigned char prefix; };
+};
+
 #endif //DEFI_MASTERNODES_ACCOUNTSHISTORY_H

src/masternodes/criminals.cpp

@@ -43,11 +43,11 @@ bool CMintedHeadersView::FetchMintedHeaders(const uint256 & txid, const uint64_t
     }
 
     blockHeaders.clear();
-    ForEach<MintedHeaders,DBMNBlockHeadersKey,CBlockHeader>([&txid, &mintedBlocks, &blockHeaders] (DBMNBlockHeadersKey const & key, CBlockHeader & blockHeader) {
+    ForEach<MintedHeaders,DBMNBlockHeadersKey,CBlockHeader>([&txid, &mintedBlocks, &blockHeaders] (DBMNBlockHeadersKey const & key, CLazySerialize<CBlockHeader> blockHeader) {
         if (key.masternodeID == txid &&
             key.mintedBlocks == mintedBlocks) {
 
-            blockHeaders.emplace(key.blockHash, std::move(blockHeader));
+            blockHeaders.emplace(key.blockHash, blockHeader.get());
             return true; // continue
         }
         return false; // break!
@@ -76,12 +76,12 @@ void CCriminalProofsView::RemoveCriminalProofs(const uint256 & mnId) {
 CCriminalProofsView::CMnCriminals CCriminalProofsView::GetUnpunishedCriminals() {
 
     CMnCriminals result;
-    ForEach<Proofs, uint256, CDoubleSignFact>([&result] (uint256 const & id, CDoubleSignFact & proof) {
+    ForEach<Proofs, uint256, CDoubleSignFact>([&result] (uint256 const & id, CLazySerialize<CDoubleSignFact> proof) {
 
         // matching with already punished. and this is the ONLY measure!
         auto node = pcustomcsview->GetMasternode(id); // assert?
         if (node && node->banTx.IsNull()) {
-            result.emplace(id, std::move(proof));
+            result.emplace(id, proof.get());
         }
         return true; // continue
     });

src/masternodes/govvariables/lp_splits.cpp

@@ -50,12 +50,12 @@ Res LP_SPLITS::Validate(const CCustomCSView & mnview) const {
 }
 
 Res LP_SPLITS::Apply(CCustomCSView & mnview) {
-    mnview.ForEachPoolPair([&] (const DCT_ID poolId, const CPoolPair & pool) {
+    mnview.ForEachPoolPair([&] (const DCT_ID poolId, CPoolPair pool) {
         // we ought to reset previous value:
-        const_cast<CPoolPair &>(pool).rewardPct = 0;
+        pool.rewardPct = 0;
         auto it = splits.find(poolId);
         if (it != splits.end()) {
-            const_cast<CPoolPair &>(pool).rewardPct = it->second;
+            pool.rewardPct = it->second;
         }
 
         mnview.SetPoolPair(poolId, pool);

src/masternodes/incentivefunding.cpp

@@ -28,9 +28,9 @@ Res CCommunityBalancesView::SetCommunityBalance(CommunityAccountType account, CA
     return Res::Ok();
 }
 
-void CCommunityBalancesView::ForEachCommunityBalance(std::function<bool (CommunityAccountType, CAmount)> callback) const
+void CCommunityBalancesView::ForEachCommunityBalance(std::function<bool (CommunityAccountType, CLazySerialize<CAmount>)> callback) const
 {
-    ForEach<ById, unsigned char, CAmount>([&callback] (unsigned char const & key, CAmount const & val) {
+    ForEach<ById, unsigned char, CAmount>([&callback] (unsigned char const & key, CLazySerialize<CAmount> val) {
         return callback(CommunityAccountCodeToType(key), val);
     }, '\0');
 

src/masternodes/incentivefunding.h

@@ -35,7 +35,7 @@ class CCommunityBalancesView : public virtual CStorageView
     CAmount GetCommunityBalance(CommunityAccountType account) const;
     Res SetCommunityBalance(CommunityAccountType account, CAmount amount);
 
-    void ForEachCommunityBalance(std::function<bool(CommunityAccountType account, CAmount amount)> callback) const;
+    void ForEachCommunityBalance(std::function<bool(CommunityAccountType, CLazySerialize<CAmount>)> callback) const;
 
     Res AddCommunityBalance(CommunityAccountType account, CAmount amount);
     Res SubCommunityBalance(CommunityAccountType account, CAmount amount);