Add in-node RPC cache, optimize getloaninfo, refactor getburninfo (#1327

)

* Add in-node RPC cache

* Change header define names

* Update comments

* Change invalidation strategy for rollback conditions, add logging

* Fix loaninfo to be better indepdent of nested calls

* Fix loan tokens

* Parallelize getloaninfo

* Reorder burn info, fix min workers

* Cleanup burn info

* Update memory ordering

src/Makefile.am

@@ -215,8 +215,9 @@ DEFI_CORE_H = \
   rpc/register.h \
   rpc/request.h \
   rpc/server.h \
-  rpc/util.h \
   rpc/stats.h \
+  rpc/resultcache.h \
+  rpc/util.h \
   scheduler.h \
   script/descriptor.h \
   script/keyorigin.h \
@@ -427,6 +428,7 @@ libdefi_server_a_SOURCES = \
   rpc/net.cpp \
   rpc/rawtransaction.cpp \
   rpc/server.cpp \
+  rpc/resultcache.cpp \
   script/sigcache.cpp \
   shutdown.cpp \
   spv/btctransaction.cpp \
@@ -603,6 +605,7 @@ libdefi_common_a_SOURCES = \
   rpc/rawtransaction_util.cpp \
   rpc/util.cpp \
   rpc/stats.cpp \
+  rpc/resultcache.cpp \
   scheduler.cpp \
   script/descriptor.cpp \
   script/sign.cpp \

src/httprpc.cpp

@@ -10,7 +10,6 @@
 #include <rpc/protocol.h>
 #include <rpc/server.h>
 #include <rpc/stats.h>
-#include <sync.h>
 #include <ui_interface.h>
 #include <util/strencodings.h>
 #include <util/system.h>

src/init.cpp

@@ -41,6 +41,7 @@
 #include <rpc/blockchain.h>
 #include <rpc/register.h>
 #include <rpc/stats.h>
+#include <rpc/resultcache.h>
 #include <rpc/server.h>
 #include <rpc/util.h>
 #include <scheduler.h>
@@ -598,6 +599,7 @@ void SetupServerArgs()
     gArgs.AddArg("-rpcallowcors=<host>", "Allow CORS requests from the given host origin. Include scheme and port (eg: -rpcallowcors=http://127.0.0.1:5000)", ArgsManager::ALLOW_ANY, OptionsCategory::RPC);
     gArgs.AddArg("-rpcstats", strprintf("Log RPC stats. (default: %u)", DEFAULT_RPC_STATS), ArgsManager::ALLOW_ANY, OptionsCategory::RPC);
     gArgs.AddArg("-consolidaterewards=<token-or-pool-symbol>", "Consolidate rewards on startup. Accepted multiple times for each token symbol", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
+    gArgs.AddArg("-rpccache=<0/1/2>", "Cache rpc results - uses additional memory to hold on to the last results per block, but faster (0=none, 1=all, 2=smart)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
 
 #if HAVE_DECL_DAEMON
     gArgs.AddArg("-daemon", "Run in the background as a daemon and accept commands", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -815,6 +817,18 @@ static bool AppInitServers()
 {
     if (!gArgs.GetBoolArg("-rpcstats", DEFAULT_RPC_STATS))
         statsRPC.setActive(false);
+
+    auto rpcCacheModeVal = gArgs.GetArg("-rpccache", 1);
+    auto rpcCacheMode = [=](){
+        switch (rpcCacheModeVal) {
+        case 1: return RPCResultCache::RPCCacheMode::All;
+        // For the moment, there is smart is dumb, just redirects to all.
+        // Future implementations could be smarter based on size / latency.
+        case 2: return RPCResultCache::RPCCacheMode::All;
+        default: return RPCResultCache::RPCCacheMode::None;
+    }}();
+    GetRPCResultCache().Init(rpcCacheMode);
+
     RPCServer::OnStarted(&OnRPCStarted);
     RPCServer::OnStopped(&OnRPCStopped);
     if (!InitHTTPServer())

src/logging.cpp

@@ -156,7 +156,8 @@ const CLogCategoryDesc LogCategories[] =
     {BCLog::LOAN, "loan"},
     {BCLog::ACCOUNTCHANGE, "accountchange"},
     {BCLog::FUTURESWAP, "futureswap"},
-    {BCLog::TOKEN_SPLIT, "tokensplit"},
+    {BCLog::TOKENSPLIT, "tokensplit"},
+    {BCLog::RPCCACHE, "rpccache"},
     {BCLog::ALL, "1"},
     {BCLog::ALL, "all"},
 };

src/logging.h

@@ -62,7 +62,9 @@ namespace BCLog {
         LOAN          = (1 << 25),
         ACCOUNTCHANGE = (1 << 26),
         FUTURESWAP    = (1 << 27),
-        TOKEN_SPLIT  = (1 << 28),
+        TOKENSPLIT    = (1 << 28),
+        RPCCACHE      = (1 << 29),
+        // Note: We're almost hitting 32 bit threshold.
         ALL           = ~(uint32_t)0,
     };
 

src/masternodes/masternodes.cpp

@@ -17,6 +17,7 @@
 #include <validation.h>
 #include <wallet/wallet.h>
 #include <wallet/walletutil.h>
+#include <rpc/resultcache.h>
 
 #include <algorithm>
 #include <functional>
@@ -591,6 +592,7 @@ int CLastHeightView::GetLastHeight() const
 
 void CLastHeightView::SetLastHeight(int height)
 {
+    SetLastValidatedHeight(height);
     Write(Height::prefix(), height);
 }
 

src/masternodes/mn_rpc.cpp

@@ -675,6 +675,7 @@ UniValue getgov(const JSONRPCRequest& request) {
                        + HelpExampleRpc("getgov", "LP_DAILY_DFI_REWARD")
                },
     }.Check(request);
+    if (auto res = GetRPCResultCache().TryGet(request)) return *res;
 
     LOCK(cs_main);
 
@@ -683,7 +684,7 @@ UniValue getgov(const JSONRPCRequest& request) {
     if (var) {
         UniValue ret(UniValue::VOBJ);
         ret.pushKV(var->GetName(),var->Export());
-        return ret;
+        return GetRPCResultCache().Set(request, ret);
     }
     throw JSONRPCError(RPC_INVALID_REQUEST, "Variable '" + name + "' not registered");
 }
@@ -712,6 +713,8 @@ UniValue listgovs(const JSONRPCRequest& request) {
                },
     }.Check(request);
 
+    if (auto res = GetRPCResultCache().TryGet(request)) return *res;
+
     GovVarsFilter mode{GovVarsFilter::All};
     std::string prefix;
     if (request.params.size() > 0) {
@@ -787,7 +790,7 @@ UniValue listgovs(const JSONRPCRequest& request) {
         result.push_back(innerResult);
     }
 
-    return result;
+    return GetRPCResultCache().Set(request, result);
 }
 
 

src/masternodes/mn_rpc.h

@@ -13,6 +13,7 @@
 
 #include <rpc/rawtransaction_util.h>
 #include <rpc/server.h>
+#include <rpc/resultcache.h>
 #include <rpc/util.h>
 
 //#ifdef ENABLE_WALLET