Merge branch 'develop' into testnet

CMakeModules/cpp_graphql_gen.cmake

@@ -6,7 +6,7 @@ set(Boost_NO_WARN_NEW_VERSIONS 1)
 FetchContent_Declare(
   cppgraphqlgen
   GIT_REPOSITORY https://github.com/microsoft/cppgraphqlgen.git
-  GIT_TAG v4.5.6
+  GIT_TAG v4.5.8
   GIT_SHALLOW TRUE
 )
 set(GRAPHQL_BUILD_TESTS OFF)

CMakeModules/cppcheck.cmake

@@ -10,6 +10,7 @@ else ()
             COMMAND ${CPP_CHECK_EXE}
             --error-exitcode=1
             --enable=all
+            --check-level=exhaustive
             --suppress=missingInclude
             --suppress=missingIncludeSystem
             # find_if - useless here
@@ -19,14 +20,20 @@ else ()
             # false positive
             --suppress=uninitMemberVar:${PROJECT_SOURCE_DIR}/*/UPnP.cpp
             # This is only enabled because of test functions and false positives
-            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/Eth.cpp
             --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/Common.cpp
             --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/Base64.cpp
             --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/util.cpp
             --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/vrf_wrapper.cpp
             --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/UPnP.cpp
             --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/logger.cpp
             --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/state_api.cpp
+            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/watches.hpp
+            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/vector_ref.h
+            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/RLP.h
+            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/FixedHash.h
+            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/CommonData.h
+            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/Common.h
+            --suppress=unusedFunction:${PROJECT_SOURCE_DIR}/*/Log.h
             # TODO remove this when we solve correct exit of programs
             --suppress=localMutex:${PROJECT_SOURCE_DIR}/*/main.cpp
             # Just style warning
@@ -38,6 +45,7 @@ else ()
             --suppress=unmatchedSuppression:${PROJECT_SOURCE_DIR}/*/Common.h
             --suppress=cstyleCast:${PROJECT_SOURCE_DIR}/*/vector_ref.h
             --suppress=cstyleCast:${PROJECT_SOURCE_DIR}/*/Common.h
+            --suppress=cstyleCast:${PROJECT_SOURCE_DIR}/*/transaction.cpp
             # not an issue here
             --suppress=virtualCallInConstructor:${PROJECT_SOURCE_DIR}/*/final_chain.cpp
             # just a warning

conanfile.py

@@ -13,14 +13,14 @@ class TaraxaConan(ConanFile):
     generators = "cmake"
 
     def requirements(self):
-        self.requires("boost/1.85.0")
-        self.requires("cppcheck/2.12") #TODO(2.14.1)
-        self.requires("openssl/3.2.1")
+        self.requires("boost/1.86.0")
+        self.requires("cppcheck/2.15.0")
+        self.requires("openssl/3.3.2")
         self.requires("cryptopp/8.9.0")
-        self.requires("gtest/1.14.0")
-        self.requires("lz4/1.9.4")
+        self.requires("gtest/1.15.0")
+        self.requires("lz4/1.10.0")
         self.requires("rocksdb/9.2.1")
-        self.requires("prometheus-cpp/1.1.0")
+        self.requires("prometheus-cpp/1.2.4")
         self.requires("jsoncpp/1.9.5")
 
     def _configure_boost_libs(self):

libraries/aleth/libdevcore/Common.h

@@ -127,33 +127,6 @@ using strings = std::vector<std::string>;
 // Null/Invalid values for convenience.
 extern bytes const NullBytes;
 
-/// Interprets @a _u as a two's complement signed number and returns the
-/// resulting s256.
-inline s256 u2s(u256 _u) {
-  static const bigint c_end = bigint(1) << 256;
-  if (boost::multiprecision::bit_test(_u, 255))
-    return s256(-(c_end - _u));
-  else
-    return s256(_u);
-}
-
-/// @returns the two's complement signed representation of the signed number _u.
-inline u256 s2u(s256 _u) {
-  static const bigint c_end = bigint(1) << 256;
-  if (_u >= 0)
-    return u256(_u);
-  else
-    return u256(c_end + _u);
-}
-
-/// @returns the smallest n >= 0 such that (1 << n) >= _x
-inline unsigned int toLog2(u256 _x) {
-  unsigned ret;
-  for (ret = 0; _x >>= 1; ++ret) {
-  }
-  return ret;
-}
-
 template <size_t n>
 inline u256 exp10() {
   return exp10<n - 1>() * u256(10);
@@ -164,12 +137,6 @@ inline u256 exp10<0>() {
   return u256(1);
 }
 
-/// @returns the absolute distance between _a and _b.
-template <class N>
-inline N diff(N const& _a, N const& _b) {
-  return std::max(_a, _b) - std::min(_a, _b);
-}
-
 /// RAII utility class whose destructor calls a given function.
 class ScopeGuard {
  public:

libraries/config/include/config/config_utils.hpp

@@ -14,7 +14,7 @@ std::string getConfigErr(const std::vector<std::string> &path);
 Json::Value getConfigData(Json::Value root, const std::vector<std::string> &path, bool optional = false);
 
 std::string getConfigDataAsString(const Json::Value &root, const std::vector<std::string> &path, bool optional = false,
-                                  std::string value = {});
+                                  const std::string &value = {});
 
 uint32_t getConfigDataAsUInt(const Json::Value &root, const std::vector<std::string> &path, bool optional = false,
                              uint32_t value = 0);

libraries/config/include/config/network.hpp

@@ -22,6 +22,8 @@ struct ConnectionConfig {
   // Number of threads dedicated to the rpc calls processing, default = 5
   uint16_t threads_num{5};
 
+  uint32_t max_pending_tasks{100};
+
   void validate() const;
 };
 

libraries/config/src/config_utils.cpp

@@ -28,7 +28,7 @@ Json::Value getConfigData(Json::Value root, const std::vector<std::string> &path
 }
 
 std::string getConfigDataAsString(const Json::Value &root, const std::vector<std::string> &path, bool optional,
-                                  std::string value) {
+                                  const std::string &value) {
   try {
     Json::Value ret = getConfigData(root, path, optional);
     if (ret.isNull()) {

libraries/config/src/network.cpp

@@ -33,6 +33,11 @@ void dec_json(const Json::Value &json, ConnectionConfig &config) {
     config.ws_port = ws_port.asUInt();
   }
 
+  // max pending tasks
+  if (auto max_pending_tasks = getConfigData(json, {"max_pending_tasks"}, true); !max_pending_tasks.isNull()) {
+    config.max_pending_tasks = max_pending_tasks.asUInt();
+  }
+
   // number of threads processing rpc calls
   if (auto threads_num = getConfigData(json, {"threads_num"}, true); !threads_num.isNull()) {
     config.threads_num = threads_num.asUInt();

libraries/core_libs/consensus/src/dag/dag.cpp

@@ -148,7 +148,7 @@ bool Dag::computeOrder(const blk_hash_t &anchor, std::vector<blk_hash_t> &ordere
       dfs.push({cur.first, true});
       std::vector<std::pair<blk_hash_t, vertex_t>> neighbors;
       // iterate through neighbors
-      for (std::tie(adj_s, adj_e) = adjacenct_vertices(cur.first, graph_); adj_s != adj_e; adj_s++) {
+      for (std::tie(adj_s, adj_e) = boost::adjacent_vertices(cur.first, graph_); adj_s != adj_e; adj_s++) {
         if (epfriend.find(index_map[*adj_s]) == epfriend.end()) {  // not in this epoch
           continue;
         }
@@ -183,7 +183,7 @@ bool Dag::reachable(vertex_t const &from, vertex_t const &to) const {
     vertex_t t = st.top();
     st.pop();
     vertex_adj_iter_t s, e;
-    for (std::tie(s, e) = adjacenct_vertices(t, graph_); s != e; ++s) {
+    for (std::tie(s, e) = boost::adjacent_vertices(t, graph_); s != e; ++s) {
       if (visited.count(*s)) continue;
       if (*s == target) return true;
       visited.insert(*s);
@@ -221,7 +221,7 @@ std::vector<blk_hash_t> PivotTree::getGhostPath(const blk_hash_t &vertex) const
     cur = st.top();
     st.pop();
     post_order.emplace_back(cur);
-    for (std::tie(s, e) = adjacenct_vertices(cur, graph_); s != e; s++) {
+    for (std::tie(s, e) = boost::adjacent_vertices(cur, graph_); s != e; s++) {
       st.emplace(*s);
     }
   }
@@ -232,7 +232,7 @@ std::vector<blk_hash_t> PivotTree::getGhostPath(const blk_hash_t &vertex) const
   for (auto const &n : post_order) {
     auto total_w = 0;
     // get childrens
-    for (std::tie(s, e) = adjacenct_vertices(n, graph_); s != e; s++) {
+    for (std::tie(s, e) = boost::adjacent_vertices(n, graph_); s != e; s++) {
       if (weight_map.count(*s)) {  // bigger timestamp
         total_w += weight_map[*s];
       }
@@ -248,7 +248,7 @@ std::vector<blk_hash_t> PivotTree::getGhostPath(const blk_hash_t &vertex) const
     size_t heavist = 0;
     vertex_t next = root;
 
-    for (std::tie(s, e) = adjacenct_vertices(root, graph_); s != e; s++) {
+    for (std::tie(s, e) = boost::adjacent_vertices(root, graph_); s != e; s++) {
       if (!weight_map.count(*s)) continue;  // bigger timestamp
       size_t w = weight_map[*s];
       assert(w > 0);
@@ -257,7 +257,6 @@ std::vector<blk_hash_t> PivotTree::getGhostPath(const blk_hash_t &vertex) const
         next = *s;
       } else if (w == heavist) {
         if (index_map[*s] < index_map[next]) {
-          heavist = w;
           next = *s;
         }
       }

libraries/core_libs/consensus/src/dag/dag_manager.cpp

@@ -375,7 +375,8 @@ uint DagManager::setDagBlockOrder(blk_hash_t const &new_anchor, PbftPeriod perio
       } else {
         db_->removeDagBlock(blk_hash);
         seen_blocks_.erase(blk_hash);
-        for (const auto &trx : dag_block->getTrxs()) expired_dag_blocks_transactions.emplace_back(trx);
+        const auto dag_trxs = dag_block->getTrxs();
+        std::copy(dag_trxs.begin(), dag_trxs.end(), std::back_inserter(expired_dag_blocks_transactions));
       }
     }
   }

libraries/core_libs/network/include/network/http_server.hpp

@@ -4,6 +4,7 @@
 #include <boost/asio.hpp>
 #include <boost/beast.hpp>
 
+#include "common/thread_pool.hpp"
 #include "common/types.hpp"
 #include "logger/logger.hpp"
 
@@ -22,15 +23,17 @@ class HttpHandler;
 
 class HttpServer : public std::enable_shared_from_this<HttpServer> {
  public:
-  HttpServer(boost::asio::io_context& io, boost::asio::ip::tcp::endpoint ep, const addr_t& node_addr,
-             const std::shared_ptr<HttpProcessor>& request_processor);
+  HttpServer(std::shared_ptr<util::ThreadPool> thread_pool, boost::asio::ip::tcp::endpoint ep, const addr_t& node_addr,
+             const std::shared_ptr<HttpProcessor>& request_processor, uint32_t max_pending_tasks);
 
   virtual ~HttpServer() { HttpServer::stop(); }
 
   bool start();
   bool stop();
 
   void accept();
+  uint32_t numberOfPendingTasks() const;
+  bool pendingTasksOverLimit() const { return numberOfPendingTasks() > kMaxPendingTasks; }
   boost::asio::io_context& getIoContext() { return io_context_; }
   std::shared_ptr<HttpServer> getShared();
   std::shared_ptr<HttpConnection> createConnection();
@@ -45,6 +48,8 @@ class HttpServer : public std::enable_shared_from_this<HttpServer> {
   boost::asio::io_context& io_context_;
   boost::asio::ip::tcp::acceptor acceptor_;
   boost::asio::ip::tcp::endpoint ep_;
+  std::weak_ptr<util::ThreadPool> thread_pool_;
+  const uint32_t kMaxPendingTasks;
   LOG_OBJECTS_DEFINE
 };
 // QQ:

libraries/core_libs/network/include/network/ws_server.hpp

@@ -16,6 +16,7 @@
 #include <thread>
 #include <vector>
 
+#include "common/thread_pool.hpp"
 #include "config/config.hpp"
 #include "dag/dag_block.hpp"
 #include "final_chain/data.hpp"
@@ -84,7 +85,8 @@ class WsSession : public std::enable_shared_from_this<WsSession> {
 // Accepts incoming connections and launches the sessions
 class WsServer : public std::enable_shared_from_this<WsServer>, public jsonrpc::AbstractServerConnector {
  public:
-  WsServer(boost::asio::io_context& ioc, tcp::endpoint endpoint, addr_t node_addr);
+  WsServer(std::shared_ptr<util::ThreadPool> thread_pool, tcp::endpoint endpoint, addr_t node_addr,
+           uint32_t max_pending_tasks);
   virtual ~WsServer();
 
   WsServer(const WsServer&) = delete;
@@ -101,6 +103,8 @@ class WsServer : public std::enable_shared_from_this<WsServer>, public jsonrpc::
   void newPendingTransaction(const trx_hash_t& trx_hash);
   void newPillarBlockData(const pillar_chain::PillarBlockData& pillar_block_data);
   uint32_t numberOfSessions();
+  uint32_t numberOfPendingTasks() const;
+  bool pendingTasksOverLimit() const { return numberOfPendingTasks() > kMaxPendingTasks; }
 
   virtual std::shared_ptr<WsSession> createSession(tcp::socket&& socket) = 0;
 
@@ -118,6 +122,8 @@ class WsServer : public std::enable_shared_from_this<WsServer>, public jsonrpc::
   boost::shared_mutex sessions_mtx_;
 
  protected:
+  std::weak_ptr<util::ThreadPool> thread_pool_;
+  uint32_t kMaxPendingTasks;
   const addr_t node_addr_;
 };
 

libraries/core_libs/network/src/http_server.cpp

@@ -2,9 +2,15 @@
 
 namespace taraxa::net {
 
-HttpServer::HttpServer(boost::asio::io_context &io, boost::asio::ip::tcp::endpoint ep, const addr_t &node_addr,
-                       const std::shared_ptr<HttpProcessor> &request_processor)
-    : request_processor_(request_processor), io_context_(io), acceptor_(io), ep_(std::move(ep)) {
+HttpServer::HttpServer(std::shared_ptr<util::ThreadPool> thread_pool, boost::asio::ip::tcp::endpoint ep,
+                       const addr_t &node_addr, const std::shared_ptr<HttpProcessor> &request_processor,
+                       uint32_t max_pending_tasks)
+    : request_processor_(request_processor),
+      io_context_(thread_pool->unsafe_get_io_context()),
+      acceptor_(thread_pool->unsafe_get_io_context()),
+      ep_(std::move(ep)),
+      thread_pool_(thread_pool),
+      kMaxPendingTasks(max_pending_tasks) {
   LOG_OBJECTS_CREATE("HTTP");
   LOG(log_si_) << "Taraxa HttpServer started at port: " << ep_.port();
 }
@@ -66,6 +72,14 @@ bool HttpServer::stop() {
   return true;
 }
 
+uint32_t HttpServer::numberOfPendingTasks() const {
+  auto thread_pool = thread_pool_.lock();
+  if (thread_pool) {
+    return thread_pool->num_pending_tasks();
+  }
+  return 0;
+}
+
 std::shared_ptr<HttpConnection> HttpConnection::getShared() {
   try {
     return shared_from_this();
@@ -98,10 +112,17 @@ void HttpConnection::read() {
         } else {
           assert(server_->request_processor_);
           LOG(server_->log_dg_) << "Received: " << request_;
-          response_ = server_->request_processor_->process(request_);
-          boost::beast::http::async_write(
-              socket_, response_,
-              [this_sp = getShared()](auto const & /*ec*/, auto /*bytes_transferred*/) { this_sp->stop(); });
+
+          if (server_->pendingTasksOverLimit()) {
+            LOG(server_->log_er_) << "HttpConnection closed - pending tasks over the limit "
+                                  << server_->numberOfPendingTasks();
+            stop();
+          } else {
+            response_ = server_->request_processor_->process(request_);
+            boost::beast::http::async_write(
+                socket_, response_,
+                [this_sp = getShared()](auto const & /*ec*/, auto /*bytes_transferred*/) { this_sp->stop(); });
+          }
         }
       });
 }

libraries/core_libs/network/src/tarcap/packets_handlers/latest/status_packet_handler.cpp

@@ -68,7 +68,7 @@ void StatusPacketHandler::process(const threadpool::PacketData& packet_data, con
     }
 
     if (genesis_hash != kGenesisHash) {
-      LOG((peers_state_->getPeersCount()) ? log_nf_ : log_er_)
+      LOG((peers_state_->getPeersCount()) ? log_nf_ : log_wr_)
           << "Incorrect genesis hash " << genesis_hash << ", host " << packet_data.from_node_id_.abridged()
           << " will be disconnected";
       disconnect(packet_data.from_node_id_, dev::p2p::UserReason);