From a88feea531806302312b4872b48b69803b5e6915 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Mon, 23 Sep 2024 16:27:12 +0200 Subject: [PATCH 1/9] bugfix: split pillar vote packet to serverl ones --- CMakeLists.txt | 2 +- .../pillar_votes_bundle_packet_handler.hpp | 2 +- ...get_pillar_votes_bundle_packet_handler.cpp | 40 ++++++++++++++----- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f6614691c1..6bb171e4e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.20) # Set current version of the project set(TARAXA_MAJOR_VERSION 1) set(TARAXA_MINOR_VERSION 11) -set(TARAXA_PATCH_VERSION 3) +set(TARAXA_PATCH_VERSION 4) set(TARAXA_VERSION ${TARAXA_MAJOR_VERSION}.${TARAXA_MINOR_VERSION}.${TARAXA_PATCH_VERSION}) # Any time a change in the network protocol is introduced this version should be increased diff --git a/libraries/core_libs/network/include/network/tarcap/packets_handlers/latest/pillar_votes_bundle_packet_handler.hpp b/libraries/core_libs/network/include/network/tarcap/packets_handlers/latest/pillar_votes_bundle_packet_handler.hpp index 550ca3e6a8..15a9ccfff7 100644 --- a/libraries/core_libs/network/include/network/tarcap/packets_handlers/latest/pillar_votes_bundle_packet_handler.hpp +++ b/libraries/core_libs/network/include/network/tarcap/packets_handlers/latest/pillar_votes_bundle_packet_handler.hpp @@ -18,7 +18,7 @@ class PillarVotesBundlePacketHandler : public ExtPillarVotePacketHandler { virtual void validatePacketRlpFormat(const threadpool::PacketData& packet_data) const override; virtual void process(const threadpool::PacketData& packet_data, const std::shared_ptr& peer) override; - protected: + public: constexpr static size_t kMaxPillarVotesInBundleRlp{250}; }; diff --git a/libraries/core_libs/network/src/tarcap/packets_handlers/latest/get_pillar_votes_bundle_packet_handler.cpp b/libraries/core_libs/network/src/tarcap/packets_handlers/latest/get_pillar_votes_bundle_packet_handler.cpp index 2c01eff1ae..cc71e189e9 100644 --- a/libraries/core_libs/network/src/tarcap/packets_handlers/latest/get_pillar_votes_bundle_packet_handler.cpp +++ b/libraries/core_libs/network/src/tarcap/packets_handlers/latest/get_pillar_votes_bundle_packet_handler.cpp @@ -1,5 +1,7 @@ #include "network/tarcap/packets_handlers/latest/get_pillar_votes_bundle_packet_handler.hpp" +#include "network/tarcap/packets_handlers/latest/pillar_votes_bundle_packet_handler.hpp" + namespace taraxa::network::tarcap { GetPillarVotesBundlePacketHandler::GetPillarVotesBundlePacketHandler( @@ -42,19 +44,39 @@ void GetPillarVotesBundlePacketHandler::process(const threadpool::PacketData &pa LOG(log_dg_) << "No pillar votes for period " << period << "and pillar block hash " << pillar_block_hash; return; } + // Check if the votes size exceeds the maximum limit and split into multiple packets if needed + const size_t total_votes = votes.size(); + size_t votes_sent = 0; - dev::RLPStream s(votes.size()); - for (const auto &sig : votes) { - s.appendRaw(sig->rlp()); - } + while (votes_sent < total_votes) { + // Determine the size of the current chunk + const size_t chunk_size = + std::min(PillarVotesBundlePacketHandler::kMaxPillarVotesInBundleRlp, total_votes - votes_sent); + + // Create a new RLPStream for the chunk + dev::RLPStream s(chunk_size); + for (size_t i = 0; i < chunk_size; ++i) { + const auto &sig = votes[votes_sent + i]; + s.appendRaw(sig->rlp()); + } + + // Seal and send the chunk to the peer + if (sealAndSend(peer->getId(), SubprotocolPacketType::PillarVotesBundlePacket, std::move(s))) { + // Mark the votes in this chunk as known + for (size_t i = 0; i < chunk_size; ++i) { + peer->markPillarVoteAsKnown(votes[votes_sent + i]->getHash()); + } - if (sealAndSend(peer->getId(), SubprotocolPacketType::PillarVotesBundlePacket, std::move(s))) { - for (const auto &vote : votes) { - peer->markPillarVoteAsKnown(vote->getHash()); + LOG(log_nf_) << "Pillar votes bundle for period " << period << ", hash " << pillar_block_hash << " sent to " + << peer->getId() << " (Chunk " + << (votes_sent / PillarVotesBundlePacketHandler::kMaxPillarVotesInBundleRlp) + 1 << "/" + << (total_votes + PillarVotesBundlePacketHandler::kMaxPillarVotesInBundleRlp - 1) / + PillarVotesBundlePacketHandler::kMaxPillarVotesInBundleRlp + << ")"; } - LOG(log_nf_) << "Pillar votes bundle for period " << period << ", hash " << pillar_block_hash << " sent to " - << peer->getId(); + // Update the votes_sent counter + votes_sent += chunk_size; } } From 42560c09879ab81ad607faf1791295a5589c0ee5 Mon Sep 17 00:00:00 2001 From: mfrankovi Date: Mon, 12 Aug 2024 09:12:32 +0200 Subject: [PATCH 2/9] chore: dag block proposal limit --- .../cli/config_jsons/default/default_config.json | 2 +- .../cli/config_jsons/devnet/devnet_config.json | 2 +- .../cli/config_jsons/mainnet/mainnet_config.json | 2 +- .../cli/config_jsons/testnet/testnet_config.json | 2 +- libraries/common/include/common/constants.hpp | 1 + .../consensus/src/dag/dag_block_proposer.cpp | 13 ++++++++++--- .../core_libs/network/include/network/network.hpp | 7 +++++++ libraries/core_libs/network/src/network.cpp | 6 ++++++ 8 files changed, 28 insertions(+), 7 deletions(-) diff --git a/libraries/cli/include/cli/config_jsons/default/default_config.json b/libraries/cli/include/cli/config_jsons/default/default_config.json index 72e9409c3f..0f72c0bc9b 100644 --- a/libraries/cli/include/cli/config_jsons/default/default_config.json +++ b/libraries/cli/include/cli/config_jsons/default/default_config.json @@ -24,7 +24,7 @@ "packets_stats_time_period_ms": 60000, "peer_max_packets_processing_time_us": 10000000, "peer_max_packets_queue_size_limit": 100000, - "max_packets_queue_size": 200000 + "max_packets_queue_size": 100 }, "listen_ip": "0.0.0.0", "listen_port": 10002, diff --git a/libraries/cli/include/cli/config_jsons/devnet/devnet_config.json b/libraries/cli/include/cli/config_jsons/devnet/devnet_config.json index 47d0713725..2f812df6f4 100644 --- a/libraries/cli/include/cli/config_jsons/devnet/devnet_config.json +++ b/libraries/cli/include/cli/config_jsons/devnet/devnet_config.json @@ -24,7 +24,7 @@ "packets_stats_time_period_ms": 60000, "peer_max_packets_processing_time_us": 10000000, "peer_max_packets_queue_size_limit": 100000, - "max_packets_queue_size": 200000 + "max_packets_queue_size": 100 }, "listen_ip": "0.0.0.0", "listen_port": 10002, diff --git a/libraries/cli/include/cli/config_jsons/mainnet/mainnet_config.json b/libraries/cli/include/cli/config_jsons/mainnet/mainnet_config.json index 66cb1b4499..f82d06c4f2 100644 --- a/libraries/cli/include/cli/config_jsons/mainnet/mainnet_config.json +++ b/libraries/cli/include/cli/config_jsons/mainnet/mainnet_config.json @@ -24,7 +24,7 @@ "packets_stats_time_period_ms": 60000, "peer_max_packets_processing_time_us": 0, "peer_max_packets_queue_size_limit": 0, - "max_packets_queue_size": 200000 + "max_packets_queue_size": 100 }, "listen_ip": "0.0.0.0", "listen_port": 10002, diff --git a/libraries/cli/include/cli/config_jsons/testnet/testnet_config.json b/libraries/cli/include/cli/config_jsons/testnet/testnet_config.json index d444117b3c..676a6428c8 100644 --- a/libraries/cli/include/cli/config_jsons/testnet/testnet_config.json +++ b/libraries/cli/include/cli/config_jsons/testnet/testnet_config.json @@ -24,7 +24,7 @@ "packets_stats_time_period_ms": 60000, "peer_max_packets_processing_time_us": 0, "peer_max_packets_queue_size_limit": 0, - "max_packets_queue_size": 200000 + "max_packets_queue_size": 100 }, "listen_ip": "0.0.0.0", "listen_port": 10002, diff --git a/libraries/common/include/common/constants.hpp b/libraries/common/include/common/constants.hpp index 5d7e4540e1..9122c34c96 100644 --- a/libraries/common/include/common/constants.hpp +++ b/libraries/common/include/common/constants.hpp @@ -29,6 +29,7 @@ const uint64_t kMinTxGas{21000}; constexpr uint32_t kMinTransactionPoolSize{30000}; constexpr uint32_t kDefaultTransactionPoolSize{200000}; +constexpr uint32_t kMaxNonFinalizedTransactions{1000000}; const size_t kV2NetworkVersion = 2; diff --git a/libraries/core_libs/consensus/src/dag/dag_block_proposer.cpp b/libraries/core_libs/consensus/src/dag/dag_block_proposer.cpp index cad7733b8c..9587b11daf 100644 --- a/libraries/core_libs/consensus/src/dag/dag_block_proposer.cpp +++ b/libraries/core_libs/consensus/src/dag/dag_block_proposer.cpp @@ -51,6 +51,11 @@ bool DagBlockProposer::proposeDagBlock() { return false; } + // Do not propose dag blocks if number of non finalized transactions is over the limit + if (trx_mgr_->getNonfinalizedTrxSize() > kMaxNonFinalizedTransactions) { + return false; + } + auto frontier = dag_mgr_->getDagFrontier(); LOG(log_dg_) << "Get frontier with pivot: " << frontier.pivot << " tips: " << frontier.tips; assert(!frontier.pivot.isZero()); @@ -183,12 +188,14 @@ void DagBlockProposer::start() { while (!stopped_) { // Blocks are not proposed if we are behind the network and still syncing auto syncing = false; + auto packets_over_the_limit = false; if (auto net = network_.lock()) { syncing = net->pbft_syncing(); + packets_over_the_limit = net->packetQueueOverLimit(); } - // Only sleep if block was not proposed or if we are syncing, if block is proposed try to propose another block - // immediately - if (syncing || !proposeDagBlock()) { + // Only sleep if block was not proposed or if we are syncing or if packets queue is over the limit, if block is + // proposed try to propose another block immediately + if (syncing || packets_over_the_limit || !proposeDagBlock()) { thisThreadSleepForMilliSeconds(min_proposal_delay); } } diff --git a/libraries/core_libs/network/include/network/network.hpp b/libraries/core_libs/network/include/network/network.hpp index 8c654eaaa7..30fd397944 100644 --- a/libraries/core_libs/network/include/network/network.hpp +++ b/libraries/core_libs/network/include/network/network.hpp @@ -77,6 +77,13 @@ class Network { */ void requestPillarBlockVotesBundle(PbftPeriod period, const blk_hash_t &pillar_block_hash); + /** + * @brief Get packets queue status + * + * @return true if packets queue is over the limit + */ + bool packetQueueOverLimit() const; + // METHODS USED IN TESTS ONLY template std::shared_ptr getSpecificHandler() const; diff --git a/libraries/core_libs/network/src/network.cpp b/libraries/core_libs/network/src/network.cpp index 9091c68571..248f773951 100644 --- a/libraries/core_libs/network/src/network.cpp +++ b/libraries/core_libs/network/src/network.cpp @@ -132,6 +132,12 @@ void Network::start() { bool Network::isStarted() { return tp_.is_running(); } +bool Network::packetQueueOverLimit() const { + auto [hp_queue_size, mp_queue_size, lp_queue_size] = packets_tp_->getQueueSize(); + auto total_size = hp_queue_size + mp_queue_size + lp_queue_size; + return total_size > kConf.network.ddos_protection.max_packets_queue_size; +} + std::list Network::getAllNodes() const { return host_->getNodes(); } size_t Network::getPeerCount() { return host_->peer_count(); } From b81ba77b8cace6f6306fea2af9689b6a4cbe36b7 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Fri, 27 Sep 2024 13:37:59 +0200 Subject: [PATCH 3/9] chore: add period to error output --- libraries/core_libs/consensus/src/pbft/pbft_manager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/core_libs/consensus/src/pbft/pbft_manager.cpp b/libraries/core_libs/consensus/src/pbft/pbft_manager.cpp index 759d656505..13a2fca473 100644 --- a/libraries/core_libs/consensus/src/pbft/pbft_manager.cpp +++ b/libraries/core_libs/consensus/src/pbft/pbft_manager.cpp @@ -1388,7 +1388,7 @@ std::shared_ptr PbftManager::identifyLeaderBlock_(PbftRound round, Pb } PbftStateRootValidation PbftManager::validateFinalChainHash(const std::shared_ptr &pbft_block) const { - auto period = pbft_block->getPeriod(); + const auto period = pbft_block->getPeriod(); const auto &pbft_block_hash = pbft_block->getBlockHash(); auto prev_final_chain_hash = final_chain_->finalChainHash(period); @@ -1397,8 +1397,8 @@ PbftStateRootValidation PbftManager::validateFinalChainHash(const std::shared_pt return PbftStateRootValidation::Missing; } if (pbft_block->getFinalChainHash() != prev_final_chain_hash) { - LOG(log_er_) << "Block " << pbft_block_hash << " state root " << pbft_block->getFinalChainHash() - << " isn't matching actual " << prev_final_chain_hash.value(); + LOG(log_er_) << "Block " << period << " hash " << pbft_block_hash << " state root " + << pbft_block->getFinalChainHash() << " isn't matching actual " << prev_final_chain_hash.value(); return PbftStateRootValidation::Invalid; } From 58ecf1507fe18e8fb93ece9ccdaefe4a1f9feca2 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Sun, 29 Sep 2024 09:38:36 +0200 Subject: [PATCH 4/9] revert change of dpos bytecode --- submodules/taraxa-evm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/taraxa-evm b/submodules/taraxa-evm index 9ca283df7f..3fd026e151 160000 --- a/submodules/taraxa-evm +++ b/submodules/taraxa-evm @@ -1 +1 @@ -Subproject commit 9ca283df7f6904be58dd71c56278b9fd9c2f53c5 +Subproject commit 3fd026e151bb74d1f22db07d6c875c2c9c91a82d From 3c1d6c1004ca0b568857c8dd0ec5ef5df3d42402 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Sun, 29 Sep 2024 21:17:43 +0200 Subject: [PATCH 5/9] update evm --- submodules/taraxa-evm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/taraxa-evm b/submodules/taraxa-evm index 3fd026e151..6364ada14d 160000 --- a/submodules/taraxa-evm +++ b/submodules/taraxa-evm @@ -1 +1 @@ -Subproject commit 3fd026e151bb74d1f22db07d6c875c2c9c91a82d +Subproject commit 6364ada14dd94ab5048a54427cff3f7b00a8d54f From 2646f52252c68096b23048d89c2f3ff7c983f586 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Mon, 30 Sep 2024 08:47:44 +0200 Subject: [PATCH 6/9] fix the tests --- submodules/taraxa-evm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/taraxa-evm b/submodules/taraxa-evm index 6364ada14d..dfcc4e7520 160000 --- a/submodules/taraxa-evm +++ b/submodules/taraxa-evm @@ -1 +1 @@ -Subproject commit 6364ada14dd94ab5048a54427cff3f7b00a8d54f +Subproject commit dfcc4e752080c229cc2dce1d53a488749a298883 From d3167b92f26ed85106b470784fbbd67a7a1e1f16 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Mon, 30 Sep 2024 09:23:04 +0200 Subject: [PATCH 7/9] update evm --- submodules/taraxa-evm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/taraxa-evm b/submodules/taraxa-evm index dfcc4e7520..41b7d6bb7d 160000 --- a/submodules/taraxa-evm +++ b/submodules/taraxa-evm @@ -1 +1 @@ -Subproject commit dfcc4e752080c229cc2dce1d53a488749a298883 +Subproject commit 41b7d6bb7da5cb65e5f126df7125df8371600f91 From 69ec027e58259f58575c76a34f2c7e9c537a4c35 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Mon, 30 Sep 2024 16:26:56 +0200 Subject: [PATCH 8/9] update HF number --- .../cli/include/cli/config_jsons/testnet/testnet_genesis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/cli/include/cli/config_jsons/testnet/testnet_genesis.json b/libraries/cli/include/cli/config_jsons/testnet/testnet_genesis.json index 71889b5bc8..658bc22702 100644 --- a/libraries/cli/include/cli/config_jsons/testnet/testnet_genesis.json +++ b/libraries/cli/include/cli/config_jsons/testnet/testnet_genesis.json @@ -154,6 +154,6 @@ "pillar_blocks_interval": 1000, "bridge_contract_address": "0xcAF2b453FE8382a4B8110356DF0508f6d71F22BF" }, - "cornus_hf_block_num": 1622000 + "cornus_hf_block_num": 1668000 } } \ No newline at end of file From 94878bfefd198d84b4a42f0ed8212817e73789df Mon Sep 17 00:00:00 2001 From: kstdl Date: Fri, 11 Oct 2024 17:54:28 +0300 Subject: [PATCH 9/9] fix: tracing execution block number --- libraries/core_libs/network/rpc/Debug.cpp | 15 +++------------ submodules/taraxa-evm | 2 +- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/libraries/core_libs/network/rpc/Debug.cpp b/libraries/core_libs/network/rpc/Debug.cpp index 11cf10699f..1f4c158f6e 100644 --- a/libraries/core_libs/network/rpc/Debug.cpp +++ b/libraries/core_libs/network/rpc/Debug.cpp @@ -3,12 +3,9 @@ #include #include -#include - #include "common/jsoncpp.hpp" #include "final_chain/state_api_data.hpp" #include "network/rpc/eth/data.hpp" -#include "pbft/pbft_manager.hpp" using namespace std; using namespace dev; @@ -17,10 +14,6 @@ using namespace taraxa; namespace taraxa::net { -inline EthBlockNumber get_ctx_block_num(EthBlockNumber block_number) { - return (block_number >= 1) ? block_number - 1 : 0; -} - Json::Value Debug::debug_traceTransaction(const std::string& transaction_hash) { Json::Value res; auto [trx, loc] = get_transaction_with_location(transaction_hash); @@ -28,8 +21,7 @@ Json::Value Debug::debug_traceTransaction(const std::string& transaction_hash) { throw std::runtime_error("Transaction not found"); } if (auto node = full_node_.lock()) { - return util::readJsonFromString( - node->getFinalChain()->trace({to_eth_trx(std::move(trx))}, get_ctx_block_num(loc->period))); + return util::readJsonFromString(node->getFinalChain()->trace({to_eth_trx(std::move(trx))}, loc->period)); } return res; } @@ -65,7 +57,7 @@ Json::Value Debug::trace_replayTransaction(const std::string& transaction_hash, } if (auto node = full_node_.lock()) { return util::readJsonFromString( - node->getFinalChain()->trace({to_eth_trx(std::move(trx))}, get_ctx_block_num(loc->period), std::move(params))); + node->getFinalChain()->trace({to_eth_trx(std::move(trx))}, loc->period, std::move(params))); } return res; } @@ -83,8 +75,7 @@ Json::Value Debug::trace_replayBlockTransactions(const std::string& block_num, c trxs.reserve(transactions->size()); std::transform(transactions->begin(), transactions->end(), std::back_inserter(trxs), [this](auto t) { return to_eth_trx(std::move(t)); }); - return util::readJsonFromString( - node->getFinalChain()->trace(std::move(trxs), get_ctx_block_num(block), std::move(params))); + return util::readJsonFromString(node->getFinalChain()->trace(std::move(trxs), block, std::move(params))); } return res; } diff --git a/submodules/taraxa-evm b/submodules/taraxa-evm index 41b7d6bb7d..77135fe103 160000 --- a/submodules/taraxa-evm +++ b/submodules/taraxa-evm @@ -1 +1 @@ -Subproject commit 41b7d6bb7da5cb65e5f126df7125df8371600f91 +Subproject commit 77135fe10352156f4266e57d9dc162e33257c48c