Merge pull request #2831 from Taraxa-project/master

Master -> dev

doc/quickstart_guide.md

@@ -37,7 +37,7 @@ TODO: add explanation of all config parameters
 There are 2 taraxa docker images:
 
 ### taraxa-builder:latest 
-Contains all dependecies required for building taraxa-node project, you can create local taraxa-builder image 
+Contains all dependencies required for building taraxa-node project, you can create local taraxa-builder image 
 by running this command in root directory:
 
     DOCKER_BUILDKIT=1 docker build --progress=plain --target builder -t taraxa-builder:latest .

for_devs/requirements.txt

@@ -1,198 +1,2 @@
-aiohttp==3.9.2
-aiosignal==1.3.1
-anyio==4.0.0
-apt-xapian-index==0.49
-argcomplete==2.0.0
-arrow==1.3.0
-asn1crypto==1.5.1
-attrs==23.1.0
-Babel==2.10.3
-bcrypt==4.0.1
-binaryornot==0.4.4
-bitarray==2.9.0
-blinker==1.6.2
-boto3==1.28.68
-botocore==1.31.68
-bottle==0.12.25
-Brotli==1.0.9
-build==0.10.0
-CacheControl==0.13.1
-certifi==2022.9.24
-cffi==1.16.0
-chardet==5.1.0
-cleo==2.0.1
 click==8.1.6
-click-configfile==0.2.3
-colorama==0.4.6
-command-not-found==0.3
-conan==1.59.0
-configparser==6.0.0
-cookiecutter==2.4.0
-crashtest==0.4.1
-cryptography==42.0.4
-cupshelpers==1.0
-cytoolz==0.12.2
-dbus-python==1.3.2
-defer==1.0.6
-distlib==0.3.7
-distro==1.8.0
-distro-info==1.5
-docker==5.0.3
-docker-compose==1.29.2
-dockerpty==0.4.1
-docopt==0.6.2
-dulwich==0.21.6
-eth-abi==5.0.1
-eth-account==0.10.0
-eth-hash==0.5.2
-eth-keyfile==0.7.0
-eth-keys==0.4.0
-eth-rlp==1.0.0
-eth-typing==3.5.2
-eth-utils==2.3.1
-fabric==2.7.1
-fasteners==0.19
-fastimport==0.9.14
-filelock==3.12.2
-frozenlist==1.4.1
-fuse-python==1.0.5
-ghp-import==2.1.0
-gpg==1.18.0
-gyp==0.1
-h11==0.14.0
-hamlet==10.0.1
-hexbytes==0.3.1
-html5lib==1.1
-httpcore==0.18.0
-httplib2==0.20.4
-httpx==0.25.0
-idna==3.7
-importlib-metadata==4.12.0
-importlib-resources==5.13.0
-iniconfig==1.1.1
-installer==0.7.0
-invoke==1.7.3
-jaraco.classes==3.2.1
-jeepney==0.8.0
-Jinja2==3.1.3
-jmespath==0.10.0
-joblib==1.2.0
-jsonpointer==2.0
-jsonschema==4.10.3
-keyring==24.2.0
-language-selector==0.1
-launchpadlib==1.11.0
-lazr.restfulclient==0.14.5
-lazr.uri==1.0.6
-livereload==2.6.3
-lockfile==0.12.2
-lru-dict==1.2.0
-lunr==0.6.2
-Markdown==3.4.4
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.1
-mat==0.13.4
-mdurl==0.1.2
-mechanize==0.4.8
-mergedeep==1.3.4
-mkdocs==1.4.2
-more-itertools==10.1.0
-msgpack==1.0.3
-multidict==6.0.4
-mutagen==1.46.0
-netifaces==0.11.0
-nltk==3.8.1
-node-semver==0.6.1
-numpy==1.24.2
-oauthlib==3.2.2
-olefile==0.46
-packaging==23.1
-paramiko==3.4.0
-parsimonious==0.9.0
-patch-ng==1.17.4
-pathlib2==2.3.7.post1
-pexpect==4.8.0
-Pillow==10.3.0
-pipx==1.2.0
-pkginfo==1.8.2
-platformdirs==3.10.0
-pluggy==1.2.0
-pluginbase==1.0.1
-poetry==1.6.1
-poetry-core==1.6.1
-protobuf==4.25.1
-psutil==5.9.4
-ptyprocess==0.7.0
-pycairo==1.24.0
-pycparser==2.21
-pycryptodome==3.19.0
-pycryptodomex==3.19.1
-pycups==2.0.1
-Pygments==2.15.1
-PyGObject==3.46.0
-pyinotify==0.9.6
-PyJWT==2.7.0
-pylev==1.4.0
-pylibacl==0.7.0
-PyNaCl==1.5.0
-pyparsing==3.1.0
-pyproject_hooks==1.0.0
-PyQt5==5.15.9
-PyQt5-sip==12.12.2
-pyrsistent==0.18.1
-pytest==6.2.5
-python-apt==2.6.0+ubuntu1
-python-dateutil==2.8.2
-python-debian==0.1.49+ubuntu2
-python-dotenv==1.0.0
-python-slugify==8.0.1
-pytz==2023.3
-pyunormalize==15.1.0
-pyxattr==0.8.1
-PyYAML==6.0
-pyyaml_env_tag==0.1
-regex==2022.10.31
-requests==2.31.0
-requests-toolbelt==1.0.0
-rich==13.3.1
-rlp==4.0.0
-s3transfer==0.7.0
-SecretStorage==3.3.3
-semver==2.13.0
-shellingham==1.5.1
-simple-term-menu==1.6.1
-simplejson==3.19.1
-six==1.16.0
-sniffio==1.3.0
-systemd-python==235
-tabulate==0.9.0
-text-unidecode==1.3
-texttable==1.6.7
-toml==0.10.2
-tomlkit==0.12.1
-toolz==0.12.0
-tornado==6.3.2
-tqdm==4.66.3
-trove-classifiers==2023.7.6
-types-python-dateutil==2.8.19.14
-typing_extensions==4.9.0
-ubuntu-advantage-tools==8001
-ubuntu-drivers-common==0.0.0
-ufw==0.36.2
-unattended-upgrades==0.1
-urllib3==1.26.16
-usb-creator==0.3.16
-userpath==1.9.0
-virtualenv==20.24.1+ds
-wadllib==1.3.6
-watchdog==3.0.0
-web3==6.12.0
-webencodings==0.5.1
-websocket-client==1.2.3
-websockets==10.4
-www-authenticate==0.9.2
-xkit==0.0.0
-yarl==1.9.4
-yt-dlp==2024.4.9
-zipp==1.0.0
+web3==6.20.1

libraries/aleth/libdevcore/RLP.h

@@ -500,8 +500,6 @@ class RLPStream {
   std::enable_if_t<is_integer<N>(Signedness::signed_only), RLPStream&> append(N const& _i) {
     bigint big_i(_i);
     if (big_i.sign() < 0) {
-      std::cerr << "negative number will be serialized as rlp list: " << _i
-                << "; it is recommended to not serialize negative numbers" << std::endl;
       appendList(1);
       big_i = boost::multiprecision::abs(big_i);
     }

libraries/aleth/libp2p/NodeTable.cpp

@@ -614,7 +614,7 @@ std::shared_ptr<NodeEntry> NodeTable::handlePingNode(bi::udp::endpoint const& _f
   std::shared_ptr<NodeEntry> sourceNodeEntry = nodeEntry(_packet.sourceid);
   if (sourceNodeEntry) {
     sourceNodeEntry->lastPongSentTime = RLPXDatagramFace::secondsSinceEpoch();
-    // We should update entrypoint the the one that node is reporting
+    // We should update entrypoint the one that node is reporting
     sourceNodeEntry->node.external_udp_port = in.source.udpPort();
   }
 

libraries/aleth/libp2p/UDP.h

@@ -226,7 +226,7 @@ void UDPSocket<Handler, MaxDatagramSize>::doWrite() {
 
 template <typename Handler, unsigned MaxDatagramSize>
 void UDPSocket<Handler, MaxDatagramSize>::disconnectWithError(boost::system::error_code _ec) {
-  // If !started and already stopped, shutdown has already occured. (EOF or
+  // If !started and already stopped, shutdown has already occurred. (EOF or
   // Operation canceled)
   if (!m_started && m_closed && !m_socket.is_open() /* todo: veirfy this logic*/) return;
 

libraries/core_libs/network/rpc/eth/LogFilter.cpp

@@ -121,16 +121,23 @@ void LogFilter::match_one(const ExtendedTransactionLocation& trx_loc, const Tran
 
 std::vector<LocalisedLogEntry> LogFilter::match_all(const FinalChain& final_chain) const {
   std::vector<LocalisedLogEntry> ret;
+
+  // to_block can't be greater than the last executed block number
+  const auto last_block_number = final_chain.last_block_number();
+  auto to_blk_n = to_block_ ? *to_block_ : last_block_number;
+  if (to_blk_n > last_block_number) {
+    to_blk_n = last_block_number;
+  }
+
   auto action = [&, this](EthBlockNumber blk_n) {
-    ExtendedTransactionLocation trx_loc{{{blk_n}, *final_chain.block_hash(blk_n)}};
+    ExtendedTransactionLocation trx_loc{{{blk_n}, final_chain.block_hash(blk_n).value()}};
     auto hashes = final_chain.transaction_hashes(trx_loc.period);
     for (const auto& hash : *hashes) {
       trx_loc.trx_hash = hash;
-      match_one(trx_loc, *final_chain.transaction_receipt(hash), [&](const auto& lle) { ret.push_back(lle); });
+      match_one(trx_loc, final_chain.transaction_receipt(hash).value(), [&](const auto& lle) { ret.push_back(lle); });
       ++trx_loc.position;
     }
   };
-  auto to_blk_n = to_block_ ? *to_block_ : final_chain.last_block_number();
   if (is_range_only_) {
     for (auto blk_n = from_block_; blk_n <= to_blk_n; ++blk_n) {
       action(blk_n);

libraries/core_libs/storage/include/storage/storage.hpp

@@ -208,8 +208,9 @@ class DbStorage : public std::enable_shared_from_this<DbStorage> {
                                                           const std::string& new_col_name, bool move_data = false);
 
   // For removal of LOG.old.* files in the database
-  void removeOldLogFiles() const;
+  void removeTempFiles() const;
   void removeFilesWithPattern(const std::string& directory, const std::regex& pattern) const;
+  void deleteTmpDirectories(const std::string& path) const;
 
   uint32_t getMajorVersion() const;
   std::unique_ptr<rocksdb::Iterator> getColumnIterator(const Column& c);

libraries/core_libs/storage/src/storage.cpp

@@ -49,7 +49,7 @@ DbStorage::DbStorage(fs::path const& path, uint32_t db_snapshot_each_n_pbft_bloc
   LOG_OBJECTS_CREATE("DBS");
 
   fs::create_directories(db_path_);
-  removeOldLogFiles();
+  removeTempFiles();
 
   rocksdb::Options options;
   options.create_missing_column_families = true;
@@ -94,10 +94,11 @@ DbStorage::DbStorage(fs::path const& path, uint32_t db_snapshot_each_n_pbft_bloc
   }
 }
 
-void DbStorage::removeOldLogFiles() const {
+void DbStorage::removeTempFiles() const {
   const std::regex filePattern("LOG\\.old\\.\\d+");
   removeFilesWithPattern(db_path_, filePattern);
   removeFilesWithPattern(state_db_path_, filePattern);
+  deleteTmpDirectories(path_);
 }
 
 void DbStorage::removeFilesWithPattern(const std::string& directory, const std::regex& pattern) const {
@@ -118,6 +119,22 @@ void DbStorage::removeFilesWithPattern(const std::string& directory, const std::
   }
 }
 
+void DbStorage::deleteTmpDirectories(const std::string& path) const {
+  try {
+    for (const auto& entry : fs::directory_iterator(path)) {
+      if (entry.is_directory()) {
+        std::string dirName = entry.path().filename().string();
+        if (dirName.size() >= 4 && dirName.substr(dirName.size() - 4) == ".tmp") {
+          fs::remove_all(entry.path());
+          LOG(log_dg_) << "Deleted: " << entry.path() << std::endl;
+        }
+      }
+    }
+  } catch (const fs::filesystem_error& e) {
+    LOG(log_er_) << "Error: " << e.what() << std::endl;
+  }
+}
+
 void DbStorage::updateDbVersions() {
   saveStatusField(StatusDbField::DbMajorVersion, TARAXA_DB_MAJOR_VERSION);
   saveStatusField(StatusDbField::DbMinorVersion, TARAXA_DB_MINOR_VERSION);