From d4c4725a0f4e5da041c60c98b2470832a176f1d8 Mon Sep 17 00:00:00 2001 From: Niven Date: Fri, 24 Nov 2023 17:52:06 +0800 Subject: [PATCH] EVM: Fix create token (#2686) * Add UTF-8 encoding validity check, limit dst20 token name length * fmt cpp * Add longer string test * Fix rust lint * Disable clippy on ffi * Specify exact clippy warn * Init vec with reserved namespace size * Fix throw error on max byte size check * Add genesis block token name tests * Remove unnecessary substr op * Fix createtoken rpc token name max length * Initial dev to port utf8 check on rust side * Fixes to use vec * Fix rust ffi for creating dst20 tokens * Rust lint * Fmt cpp * Revert token validation pipeline * Add unit test for rust UTF-8 encoding validity test * Fix rust lint * Switch to 31 limit * Fix dst20 test * Remove clippy ignore * Fix merge error * Expose rust str utf8 check with ffi * Fix rust lint * Remove import * Remove import * Add next upgrade fork height on tests * Disable lint checks on dst20 migration tokens * Fix rust lint --------- Co-authored-by: Jouzo <15011228+Jouzo@users.noreply.github.com> --- lib/ain-cpp-imports/src/bridge.rs | 3 +- lib/ain-cpp-imports/src/lib.rs | 10 +- lib/ain-evm/src/contract.rs | 7 +- lib/ain-evm/tests/utf8_string_test.rs | 55 +++++++++++ lib/ain-rs-exports/src/evm.rs | 12 +-- lib/ain-rs-exports/src/lib.rs | 59 +++++++----- lib/ain-rs-exports/src/util.rs | 13 +++ src/Makefile.test.include | 1 + src/dfi/consensus/tokens.cpp | 2 +- src/dfi/rpc_tokens.cpp | 3 +- src/dfi/tokens.cpp | 32 +++++-- src/dfi/tokens.h | 3 +- src/ffi/ffiexports.cpp | 26 +++++- src/ffi/ffiexports.h | 2 +- src/ffi/ffihelpers.h | 5 + src/test/utf8_string_tests.cpp | 91 +++++++++++++++++++ src/util/strencodings.cpp | 59 ++++++++++++ src/util/strencodings.h | 1 + test/functional/feature_address_map.py | 2 + test/functional/feature_dusd_loans.py | 1 + test/functional/feature_evm.py | 1 + .../feature_evm_contract_env_vars.py | 1 + test/functional/feature_evm_contracts.py | 1 + test/functional/feature_evm_dfi_intrinsics.py | 1 + test/functional/feature_evm_dst20.py | 72 ++++++++++++++- test/functional/feature_evm_eip1559_fees.py | 1 + test/functional/feature_evm_fee.py | 1 + test/functional/feature_evm_gas.py | 1 + test/functional/feature_evm_genesis.py | 2 + test/functional/feature_evm_logs.py | 1 + test/functional/feature_evm_mempool.py | 1 + test/functional/feature_evm_miner.py | 1 + test/functional/feature_evm_proxy.py | 1 + test/functional/feature_evm_rollback.py | 1 + test/functional/feature_evm_rpc.py | 1 + .../functional/feature_evm_rpc_fee_history.py | 1 + test/functional/feature_evm_rpc_filters.py | 1 + .../functional/feature_evm_rpc_transaction.py | 1 + test/functional/feature_evm_smart_contract.py | 1 + .../feature_evm_transaction_replacement.py | 1 + test/functional/feature_evm_transferdomain.py | 1 + test/functional/feature_evm_vmmap_rpc.py | 1 + test/functional/feature_icx_orderbook.py | 2 + ...re_on_chain_government_voting_scenarios.py | 1 + test/functional/rpc_blockchain.py | 1 + test/functional/rpc_updatemasternode.py | 1 + 46 files changed, 432 insertions(+), 54 deletions(-) create mode 100644 lib/ain-evm/tests/utf8_string_test.rs create mode 100644 lib/ain-rs-exports/src/util.rs create mode 100644 src/test/utf8_string_tests.cpp diff --git a/lib/ain-cpp-imports/src/bridge.rs b/lib/ain-cpp-imports/src/bridge.rs index fb1f9387890..53d3662157b 100644 --- a/lib/ain-cpp-imports/src/bridge.rs +++ b/lib/ain-cpp-imports/src/bridge.rs @@ -46,7 +46,8 @@ pub mod ffi { fn getEthSyncStatus() -> [i64; 2]; fn getAttributeValues(mnview_ptr: usize) -> Attributes; fn CppLogPrintf(message: String); - fn getDST20Tokens(mnview_ptr: usize) -> Vec; + #[allow(clippy::ptr_arg)] + fn getDST20Tokens(mnview_ptr: usize, tokens: &mut Vec) -> bool; fn getClientVersion() -> String; fn getNumCores() -> i32; fn getCORSAllowedOrigin() -> String; diff --git a/lib/ain-cpp-imports/src/lib.rs b/lib/ain-cpp-imports/src/lib.rs index 5468ac78270..fd008bd4bcd 100644 --- a/lib/ain-cpp-imports/src/lib.rs +++ b/lib/ain-cpp-imports/src/lib.rs @@ -86,7 +86,8 @@ mod ffi { // Just the logs are skipped. } - pub fn getDST20Tokens(_mnview_ptr: usize) -> Vec { + #[allow(clippy::ptr_arg)] + pub fn getDST20Tokens(_mnview_ptr: usize, _tokens: &mut Vec) -> bool { unimplemented!("{}", UNIMPL_MSG) } pub fn getClientVersion() -> String { @@ -238,9 +239,10 @@ pub fn log_print(message: &str) { ffi::CppLogPrintf(message.to_owned()); } -/// Fetches all DST20 tokens in view. -pub fn get_dst20_tokens(mnview_ptr: usize) -> Vec { - ffi::getDST20Tokens(mnview_ptr) +/// Fetches all DST20 tokens in view, returns the result of the migration +#[allow(clippy::ptr_arg)] +pub fn get_dst20_tokens(mnview_ptr: usize, tokens: &mut Vec) -> bool { + ffi::getDST20Tokens(mnview_ptr, tokens) } /// Returns the number of CPU cores available to the node. diff --git a/lib/ain-evm/src/contract.rs b/lib/ain-evm/src/contract.rs index ec53a178a4f..f5c2f853a07 100644 --- a/lib/ain-evm/src/contract.rs +++ b/lib/ain-evm/src/contract.rs @@ -411,8 +411,13 @@ fn get_default_successful_receipt() -> ReceiptV3 { } pub fn get_dst20_migration_txs(mnview_ptr: usize) -> Result> { + let mut tokens = vec![]; let mut txs = Vec::new(); - for token in ain_cpp_imports::get_dst20_tokens(mnview_ptr) { + if !ain_cpp_imports::get_dst20_tokens(mnview_ptr, &mut tokens) { + return Err(format_err!("DST20 token migration failed, invalid token name.").into()); + } + + for token in tokens { let address = ain_contracts::dst20_address_from_token_id(token.id)?; trace!( "[get_dst20_migration_txs] Deploying to address {:#?}", diff --git a/lib/ain-evm/tests/utf8_string_test.rs b/lib/ain-evm/tests/utf8_string_test.rs new file mode 100644 index 00000000000..403eac395a6 --- /dev/null +++ b/lib/ain-evm/tests/utf8_string_test.rs @@ -0,0 +1,55 @@ +#[cfg(test)] +use std::str; + +#[test] +fn check_for_valid_utf8_strings() { + let test1 = "abcdefghijklmnopqrstuvwxyz1234567890~_= ^+%]{}"; + let test2 = "abcdeàèéìòù"; + let test3 = "😁 Beaming Face With Smiling Eyes"; + let test4 = "Slightly Smiling Face 🙂"; + let test5 = "🤣🤣🤣 Rolling on the Floor Laughing"; + let test6 = "🤩🤩🤩 Star-🤩Struck 🤩🤩"; + let test7 = "Left till here away at to whom past. Feelings laughing at no wondered repeated provided finished. \ + It acceptance thoroughly my advantages everything as. Are projecting inquietude affronting preference saw who. \ + Marry of am do avoid ample as. Old disposal followed she ignorant desirous two has. Called played entire roused \ + though for one too. He into walk roof made tall cold he. Feelings way likewise addition wandered contempt bed \ + indulged."; + + let test1_bytes = test1.as_bytes(); + let test2_bytes = test2.as_bytes(); + let test3_bytes = test3.as_bytes(); + let test4_bytes = test4.as_bytes(); + let test5_bytes = test5.as_bytes(); + let test6_bytes = test6.as_bytes(); + let test7_bytes = test7.as_bytes(); + + assert_eq!(str::from_utf8(test1_bytes), Ok(test1)); + assert_eq!(str::from_utf8(test2_bytes), Ok(test2)); + assert_eq!(str::from_utf8(test3_bytes), Ok(test3)); + assert_eq!(str::from_utf8(test4_bytes), Ok(test4)); + assert_eq!(str::from_utf8(test5_bytes), Ok(test5)); + assert_eq!(str::from_utf8(test6_bytes), Ok(test6)); + assert_eq!(str::from_utf8(test7_bytes), Ok(test7)); +} + +#[test] +fn check_for_invalid_utf8_strings() { + let smiling_face = "😁".as_bytes(); + let laughing_face = "🤣".as_bytes(); + let star_struck_face = "🤩".as_bytes(); + + let mut test1 = smiling_face[0..1].to_vec(); + test1.extend_from_slice(" Beaming Face With Smiling Eyes".as_bytes()); + + let mut test2 = laughing_face[0..3].to_vec(); + test2.extend_from_slice(&laughing_face[0..2]); + test2.extend_from_slice(&laughing_face[0..1]); + test2.extend_from_slice(" Rolling on the Floor Laughing".as_bytes()); + + let mut test3 = star_struck_face[0..1].to_vec(); + test3.extend_from_slice("🤩🤩 Star-🤩Struck 🤩🤩".as_bytes()); + + assert!(str::from_utf8(test1.as_slice()).is_err()); + assert!(str::from_utf8(test2.as_slice()).is_err()); + assert!(str::from_utf8(test3.as_slice()).is_err()); +} diff --git a/lib/ain-rs-exports/src/evm.rs b/lib/ain-rs-exports/src/evm.rs index 76377af09ac..cf44382ffa2 100644 --- a/lib/ain-rs-exports/src/evm.rs +++ b/lib/ain-rs-exports/src/evm.rs @@ -726,19 +726,17 @@ fn evm_try_get_tx_by_hash(tx_hash: &str) -> Result { fn evm_try_unsafe_create_dst20( template: &mut BlockTemplateWrapper, native_hash: &str, - name: &str, - symbol: &str, - token_id: u64, + token: ffi::DST20TokenInfo, ) -> Result<()> { let native_hash = XHash::from(native_hash); - let address = ain_contracts::dst20_address_from_token_id(token_id)?; + let address = ain_contracts::dst20_address_from_token_id(token.id)?; debug!("Deploying to address {:#?}", address); let system_tx = ExecuteTx::SystemTx(SystemTx::DeployContract(DeployContractData { - name: String::from(name), - symbol: String::from(symbol), + name: token.name, + symbol: token.symbol, address, - token_id, + token_id: token.id, })); unsafe { diff --git a/lib/ain-rs-exports/src/lib.rs b/lib/ain-rs-exports/src/lib.rs index f6b306d7e07..173dc9459f0 100644 --- a/lib/ain-rs-exports/src/lib.rs +++ b/lib/ain-rs-exports/src/lib.rs @@ -1,10 +1,11 @@ mod core; mod evm; mod prelude; +mod util; use ain_evm::blocktemplate::BlockTemplate; -use crate::{core::*, evm::*}; +use crate::{core::*, evm::*, util::*}; pub struct BlockTemplateWrapper(Option); @@ -22,6 +23,32 @@ impl BlockTemplateWrapper { #[cxx::bridge] pub mod ffi { + // ========= FFI ========== + pub struct CrossBoundaryResult { + pub ok: bool, + pub reason: String, + } + + // ========= Util ========== + extern "Rust" { + fn rs_try_from_utf8(result: &mut CrossBoundaryResult, string: &'static [u8]) -> String; + } + + // ========= Core ========== + extern "Rust" { + fn ain_rs_preinit(result: &mut CrossBoundaryResult); + fn ain_rs_init_logging(result: &mut CrossBoundaryResult); + fn ain_rs_init_core_services(result: &mut CrossBoundaryResult); + fn ain_rs_wipe_evm_folder(result: &mut CrossBoundaryResult); + fn ain_rs_stop_core_services(result: &mut CrossBoundaryResult); + + // Networking + fn ain_rs_init_network_json_rpc_service(result: &mut CrossBoundaryResult, addr: &str); + fn ain_rs_init_network_grpc_service(result: &mut CrossBoundaryResult, addr: &str); + fn ain_rs_init_network_subscriptions_service(result: &mut CrossBoundaryResult, addr: &str); + fn ain_rs_stop_network_services(result: &mut CrossBoundaryResult); + } + // ========== Block ========== #[derive(Default)] pub struct EVMBlockHeader { @@ -74,26 +101,6 @@ pub mod ffi { pub key_id: u32, } - // ========= Core ========== - pub struct CrossBoundaryResult { - pub ok: bool, - pub reason: String, - } - - extern "Rust" { - fn ain_rs_preinit(result: &mut CrossBoundaryResult); - fn ain_rs_init_logging(result: &mut CrossBoundaryResult); - fn ain_rs_init_core_services(result: &mut CrossBoundaryResult); - fn ain_rs_wipe_evm_folder(result: &mut CrossBoundaryResult); - fn ain_rs_stop_core_services(result: &mut CrossBoundaryResult); - - // Networking - fn ain_rs_init_network_json_rpc_service(result: &mut CrossBoundaryResult, addr: &str); - fn ain_rs_init_network_grpc_service(result: &mut CrossBoundaryResult, addr: &str); - fn ain_rs_init_network_subscriptions_service(result: &mut CrossBoundaryResult, addr: &str); - fn ain_rs_stop_network_services(result: &mut CrossBoundaryResult); - } - // ========== EVM ========== pub struct CreateTransactionContext<'a> { @@ -129,6 +136,12 @@ pub mod ffi { pub token_id: u32, } + pub struct DST20TokenInfo { + pub id: u64, + pub name: String, + pub symbol: String, + } + #[derive(Default)] pub struct CreateTxResult { pub tx: Vec, @@ -291,9 +304,7 @@ pub mod ffi { result: &mut CrossBoundaryResult, block_template: &mut BlockTemplateWrapper, native_hash: &str, - name: &str, - symbol: &str, - token_id: u64, + token: DST20TokenInfo, ); fn evm_try_unsafe_bridge_dst20( diff --git a/lib/ain-rs-exports/src/util.rs b/lib/ain-rs-exports/src/util.rs new file mode 100644 index 00000000000..e1df8ae0505 --- /dev/null +++ b/lib/ain-rs-exports/src/util.rs @@ -0,0 +1,13 @@ +use std::str; + +use ain_evm::Result; +use ain_macros::ffi_fallible; + +use crate::{ffi, prelude::*}; + +/// Validates a slice of bytes is valid UTF-8 and converts the bytes to a rust string slice. +#[ffi_fallible] +pub fn rs_try_from_utf8(string: &'static [u8]) -> Result { + let string = str::from_utf8(string).map_err(|_| "Error interpreting bytes, invalid UTF-8")?; + Ok(string.to_string()) +} diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 3cb222b48e6..f0ef44b7bb6 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -154,6 +154,7 @@ DEFI_TESTS =\ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ + test/utf8_string_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ test/versionbits_tests.cpp \ diff --git a/src/dfi/consensus/tokens.cpp b/src/dfi/consensus/tokens.cpp index 9a9daf7de55..f84f9bc69f5 100644 --- a/src/dfi/consensus/tokens.cpp +++ b/src/dfi/consensus/tokens.cpp @@ -116,7 +116,7 @@ Res CTokensConsensus::operator()(const CCreateTokenMessage &obj) const { } } - auto tokenId = mnview.CreateToken(token, static_cast(height) < consensus.DF2BayfrontHeight, &blockCtx); + auto tokenId = mnview.CreateToken(token, static_cast(height), &blockCtx); return tokenId; } diff --git a/src/dfi/rpc_tokens.cpp b/src/dfi/rpc_tokens.cpp index c512c83bf65..0ee1c3d001a 100644 --- a/src/dfi/rpc_tokens.cpp +++ b/src/dfi/rpc_tokens.cpp @@ -26,7 +26,8 @@ UniValue createtoken(const JSONRPCRequest &request) { {"name", RPCArg::Type::STR, RPCArg::Optional::OMITTED, - "Token's name (optional), no longer than " + std::to_string(CToken::MAX_TOKEN_NAME_LENGTH)}, + "Token's name (optional), no longer than " + + std::to_string(CToken::POST_METACHAIN_TOKEN_NAME_BYTE_SIZE)}, {"isDAT", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED, diff --git a/src/dfi/tokens.cpp b/src/dfi/tokens.cpp index 8a5f96459c6..80aa5fdee1b 100644 --- a/src/dfi/tokens.cpp +++ b/src/dfi/tokens.cpp @@ -68,9 +68,7 @@ Res CTokensView::CreateDFIToken() { return Res::Ok(); } -ResVal CTokensView::CreateToken(const CTokensView::CTokenImpl &token, - bool isPreBayfront, - BlockContext *blockCtx) { +ResVal CTokensView::CreateToken(const CTokensView::CTokenImpl &token, int height, BlockContext *blockCtx) { if (GetTokenByCreationTx(token.creationTx)) { return Res::Err("token with creation tx %s already exists!", token.creationTx.ToString()); } @@ -93,7 +91,7 @@ ResVal CTokensView::CreateToken(const CTokensView::CTokenImpl &token, }, id); if (id == DCT_ID_START) { - if (isPreBayfront) { + if (height < Params().GetConsensus().DF2BayfrontHeight) { return Res::Err("Critical fault: trying to create DCT_ID same as DCT_ID_START for Foundation owner\n"); } @@ -107,12 +105,32 @@ ResVal CTokensView::CreateToken(const CTokensView::CTokenImpl &token, const auto &evmTemplate = blockCtx->GetEVMTemplate(); if (shouldCreateDst20 && evmTemplate) { CrossBoundaryResult result; + rust::string token_name{}; + rust::string token_symbol{}; + if (height >= Params().GetConsensus().DF23UpgradeHeight) { + if (token.name.size() > CToken::POST_METACHAIN_TOKEN_NAME_BYTE_SIZE) { + return Res::Err("Error creating DST20 token, token name is larger than max bytes\n"); + } + token_name = rs_try_from_utf8(result, ffi_from_string_to_slice(token.name)); + if (!result.ok) { + return Res::Err("Error creating DST20 token, token name not valid UTF-8\n"); + } + token_symbol = rs_try_from_utf8(result, ffi_from_string_to_slice(token.symbol)); + if (!result.ok) { + return Res::Err("Error creating DST20 token, token symbol not valid UTF-8\n"); + } + } else { + token_name = rust::string(token.name); + token_symbol = rust::string(token.symbol); + } evm_try_unsafe_create_dst20(result, evmTemplate->GetTemplate(), token.creationTx.GetHex(), - rust::string(token.name.c_str()), - rust::string(token.symbol.c_str()), - id.v); + DST20TokenInfo{ + id.v, + token_name, + token_symbol, + }); if (!result.ok) { return Res::Err("Error creating DST20 token: %s", result.reason); } diff --git a/src/dfi/tokens.h b/src/dfi/tokens.h index 8208ca46360..417f2d4c45d 100644 --- a/src/dfi/tokens.h +++ b/src/dfi/tokens.h @@ -24,6 +24,7 @@ class CToken { static const uint8_t MAX_TOKEN_NAME_LENGTH = 128; static const uint8_t MAX_TOKEN_SYMBOL_LENGTH = 8; static const uint8_t MAX_TOKEN_POOLPAIR_LENGTH = 16; + static const uint8_t POST_METACHAIN_TOKEN_NAME_BYTE_SIZE = 30; enum class TokenFlags : uint8_t { None = 0, Mintable = 0x01, @@ -197,7 +198,7 @@ class CTokensView : public virtual CStorageView { DCT_ID const &start = DCT_ID{0}); Res CreateDFIToken(); - ResVal CreateToken(const CTokenImpl &token, bool isPreBayfront = false, BlockContext *blockCtx = nullptr); + ResVal CreateToken(const CTokenImpl &token, int height, BlockContext *blockCtx = nullptr); Res UpdateToken(const CTokenImpl &newToken, bool isPreBayfront = false, const bool tokenSplitUpdate = false); Res BayfrontFlagsCleanup(); diff --git a/src/ffi/ffiexports.cpp b/src/ffi/ffiexports.cpp index 9e266e5bb02..c9a60052efb 100644 --- a/src/ffi/ffiexports.cpp +++ b/src/ffi/ffiexports.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -313,22 +314,39 @@ uint32_t getEthMaxResponseByteSize() { return max_response_size_mb * 1024 * 1024; } -rust::vec getDST20Tokens(std::size_t mnview_ptr) { +bool getDST20Tokens(std::size_t mnview_ptr, rust::vec &tokens) { LOCK(cs_main); - rust::vec tokens; + bool res = true; CCustomCSView *cache = reinterpret_cast(static_cast(mnview_ptr)); cache->ForEachToken( [&](DCT_ID const &id, CTokensView::CTokenImpl token) { if (!token.IsDAT() || token.IsPoolShare()) { return true; } + if (token.name.size() > CToken::POST_METACHAIN_TOKEN_NAME_BYTE_SIZE) { + res = false; + return false; + } - tokens.push_back({id.v, token.name, token.symbol}); + CrossBoundaryResult result; + auto token_name = rs_try_from_utf8(result, ffi_from_string_to_slice(token.name)); + if (!result.ok) { + LogPrintf("Error migrating DST20 token, token name not valid UTF-8\n"); + res = false; + return false; + } + auto token_symbol = rs_try_from_utf8(result, ffi_from_string_to_slice(token.symbol)); + if (!result.ok) { + LogPrintf("Error migrating DST20 token, token symbol not valid UTF-8\n"); + res = false; + return false; + } + tokens.push_back({id.v, token_name, token_symbol}); return true; }, DCT_ID{1}); // start from non-DFI - return tokens; + return res; } int32_t getNumCores() { diff --git a/src/ffi/ffiexports.h b/src/ffi/ffiexports.h index 0e8ce0511e8..9e247eece76 100644 --- a/src/ffi/ffiexports.h +++ b/src/ffi/ffiexports.h @@ -78,7 +78,7 @@ rust::string getStateInputJSON(); std::array getEthSyncStatus(); Attributes getAttributeValues(std::size_t mnview_ptr); void CppLogPrintf(rust::string message); -rust::vec getDST20Tokens(std::size_t mnview_ptr); +bool getDST20Tokens(std::size_t mnview_ptr, rust::vec &tokens); rust::string getClientVersion(); int32_t getNumCores(); rust::string getCORSAllowedOrigin(); diff --git a/src/ffi/ffihelpers.h b/src/ffi/ffihelpers.h index 56bcb04d521..3954421cc99 100644 --- a/src/ffi/ffihelpers.h +++ b/src/ffi/ffihelpers.h @@ -3,6 +3,7 @@ #include #include +#include #define XResultThrowOnErr(x) \ [&]() { \ @@ -55,4 +56,8 @@ return ResVal(std::move(res), Res::Ok()); \ }(); +inline rust::slice ffi_from_string_to_slice(const std::string &str) { + return rust::slice(reinterpret_cast(str.c_str()), str.size()); +} + #endif // DEFI_FFI_FFIHELPERS_H diff --git a/src/test/utf8_string_tests.cpp b/src/test/utf8_string_tests.cpp new file mode 100644 index 00000000000..a1016df9a18 --- /dev/null +++ b/src/test/utf8_string_tests.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(utf8_string_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(check_for_valid_utf8_strings) +{ + std::string test1 = "abcdefghijklmnopqrstuvwxyz1234567890~_= ^+%]{}"; + std::string test2 = "abcdeàèéìòù"; + std::string test3 = "😁 Beaming Face With Smiling Eyes"; + std::string test4 = "Slightly Smiling Face 🙂"; + std::string test5 = "🤣🤣🤣 Rolling on the Floor Laughing"; + std::string test6 = "🤩🤩🤩 Star-🤩Struck 🤩🤩"; + std::string test7 = "Left till here away at to whom past. Feelings laughing at no wondered repeated provided finished." + " It acceptance thoroughly my advantages everything as. Are projecting inquietude affronting preference saw who." + " Marry of am do avoid ample as. Old disposal followed she ignorant desirous two has. Called played entire roused" + " though for one too. He into walk roof made tall cold he. Feelings way likewise addition wandered contempt bed indulged."; + + // Check UTF-8 validity with CPP + BOOST_CHECK(check_is_valid_utf8(test1)); + BOOST_CHECK(check_is_valid_utf8(test2)); + BOOST_CHECK(check_is_valid_utf8(test3)); + BOOST_CHECK(check_is_valid_utf8(test4)); + BOOST_CHECK(check_is_valid_utf8(test5)); + BOOST_CHECK(check_is_valid_utf8(test6)); + BOOST_CHECK(check_is_valid_utf8(test7)); + + // Check UTF-8 validity with Rust FFI + CrossBoundaryResult result; + auto test1_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test1)); + BOOST_CHECK(result.ok); + BOOST_CHECK(test1_res == test1); + auto test2_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test2)); + BOOST_CHECK(result.ok); + BOOST_CHECK(test2_res == test2); + auto test3_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test3)); + BOOST_CHECK(result.ok); + BOOST_CHECK(test3_res == test3); + auto test4_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test4)); + BOOST_CHECK(result.ok); + BOOST_CHECK(test4_res == test4); + auto test5_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test5)); + BOOST_CHECK(result.ok); + BOOST_CHECK(test5_res == test5); + auto test6_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test6)); + BOOST_CHECK(result.ok); + BOOST_CHECK(test6_res == test6); + auto test7_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test7)); + BOOST_CHECK(result.ok); + BOOST_CHECK(test7_res == test7); + + // Check UTF +} + +BOOST_AUTO_TEST_CASE(check_for_invalid_utf8_strings) +{ + std::string smiling_face = "😁"; + std::string laughing_face = "🤣"; + std::string star_struck_face = "🤩"; + std::string test1 = smiling_face.substr(0, 1) + " Beaming Face With Smiling Eyes"; + std::string test2 = laughing_face.substr(0, 3) + laughing_face.substr(0, 2) + laughing_face.substr(0, 1) + " Rolling on the Floor Laughing"; + std::string test3 = star_struck_face.substr(0, 1) + "🤩🤩 Star-🤩Struck 🤩🤩"; + + // Check UTF-8 validity with CPP + BOOST_CHECK(!check_is_valid_utf8(test1)); + BOOST_CHECK(!check_is_valid_utf8(test2)); + BOOST_CHECK(!check_is_valid_utf8(test3)); + + // Check UTF-8 validity with Rust FFI + CrossBoundaryResult result; + auto test1_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test1)); + BOOST_CHECK(!result.ok); + BOOST_CHECK(test1_res == ""); + auto test2_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test2)); + BOOST_CHECK(!result.ok); + BOOST_CHECK(test2_res == ""); + auto test3_res = rs_try_from_utf8(result, ffi_from_string_to_slice(test3)); + BOOST_CHECK(!result.ok); + BOOST_CHECK(test3_res == ""); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 03ad0fd56b1..c6e2c3a7f13 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -578,3 +578,62 @@ std::string trim_ws(std::string const & str) size_t last = str.find_last_not_of(ws); return str.substr(first, (last - first + 1)); } + +bool check_is_valid_utf8(const std::string& str) +{ + if (!str.data()) + return true; + + const unsigned char *bytes = (const unsigned char *)str.data(); + unsigned int cp; + int num; + + while (*bytes != 0x00) + { + if ((*bytes & 0x80) == 0x00) + { + // U+0000 to U+007F + cp = (*bytes & 0x7F); + num = 1; + } + else if ((*bytes & 0xE0) == 0xC0) + { + // U+0080 to U+07FF + cp = (*bytes & 0x1F); + num = 2; + } + else if ((*bytes & 0xF0) == 0xE0) + { + // U+0800 to U+FFFF + cp = (*bytes & 0x0F); + num = 3; + } + else if ((*bytes & 0xF8) == 0xF0) + { + // U+10000 to U+10FFFF + cp = (*bytes & 0x07); + num = 4; + } + else + return false; + + bytes += 1; + for (int i = 1; i < num; ++i) + { + if ((*bytes & 0xC0) != 0x80) + return false; + cp = (cp << 6) | (*bytes & 0x3F); + bytes += 1; + } + + if ((cp > 0x10FFFF) || + ((cp >= 0xD800) && (cp <= 0xDFFF)) || + ((cp <= 0x007F) && (num != 1)) || + ((cp >= 0x0080) && (cp <= 0x07FF) && (num != 2)) || + ((cp >= 0x0800) && (cp <= 0xFFFF) && (num != 3)) || + ((cp >= 0x10000) && (cp <= 0x1FFFFF) && (num != 4))) + return false; + } + + return true; +} diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 39e179638f7..18b962db377 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -40,6 +40,7 @@ std::vector ParseHex(const char* psz); std::vector ParseHex(const std::string& str); signed char HexDigit(char c); std::string trim_ws(std::string const & str); +bool check_is_valid_utf8(const std::string& str); /* Returns true if each character in str is a hex character, and has an even * number of hex digits.*/ bool IsHex(const std::string& str); diff --git a/test/functional/feature_address_map.py b/test/functional/feature_address_map.py index 49abe79d514..085e85192b4 100755 --- a/test/functional/feature_address_map.py +++ b/test/functional/feature_address_map.py @@ -36,6 +36,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], [ @@ -53,6 +54,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_dusd_loans.py b/test/functional/feature_dusd_loans.py index edbfbfbeb3d..60b4898825f 100755 --- a/test/functional/feature_dusd_loans.py +++ b/test/functional/feature_dusd_loans.py @@ -54,6 +54,7 @@ def set_test_params(self): f"-grandcentralheight={self.grandcentralheight}", f"-grandcentralepilogueheight={self.grandcentralepilogueheight}", f"-metachainheight={self.metachainheight}", + f"-df23upgradeheight={self.metachainheight}", "-jellyfish_regtest=1", "-simulatemainnet=1", ] diff --git a/test/functional/feature_evm.py b/test/functional/feature_evm.py index dbf6fe8f3e9..d7ffe45883c 100755 --- a/test/functional/feature_evm.py +++ b/test/functional/feature_evm.py @@ -39,6 +39,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", "-ethdebug=1", ] diff --git a/test/functional/feature_evm_contract_env_vars.py b/test/functional/feature_evm_contract_env_vars.py index de23c87fc8c..58f07863b3d 100755 --- a/test/functional/feature_evm_contract_env_vars.py +++ b/test/functional/feature_evm_contract_env_vars.py @@ -32,6 +32,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_contracts.py b/test/functional/feature_evm_contracts.py index 99e0248f5ce..585eed2668f 100755 --- a/test/functional/feature_evm_contracts.py +++ b/test/functional/feature_evm_contracts.py @@ -34,6 +34,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_dfi_intrinsics.py b/test/functional/feature_evm_dfi_intrinsics.py index bd8e4370f7d..cfb04eb75ed 100755 --- a/test/functional/feature_evm_dfi_intrinsics.py +++ b/test/functional/feature_evm_dfi_intrinsics.py @@ -35,6 +35,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_dst20.py b/test/functional/feature_evm_dst20.py index 83d586eb0b9..15d2f956f13 100755 --- a/test/functional/feature_evm_dst20.py +++ b/test/functional/feature_evm_dst20.py @@ -41,11 +41,66 @@ def set_test_params(self): "-fortcanninggreatworldheight=94", "-fortcanningepilogueheight=96", "-grandcentralheight=101", - "-metachainheight=105", + "-metachainheight=153", + "-df23upgradeheight=153", "-subsidytest=1", ] ] + def test_invalid_too_long_token_name_dst20_migration_tx(self): + block_height = self.nodes[0].getblockcount() + + self.node.createtoken( + { + "symbol": "TooLongTokenName", + "name": "TheTokenWithNameMore30ByteLimit", # 31 bytes + "isDAT": True, + "collateralAddress": self.address, + } + ) + self.nodes[0].generate(2) + + # enable EVM, transferdomain, DVM to EVM transfers and EVM to DVM transfers + self.nodes[0].setgov( + { + "ATTRIBUTES": { + "v0/params/feature/evm": "true", + } + } + ) + self.nodes[0].generate(1) + + # Trigger EVM genesis DST20 migration + assert_equal(self.nodes[0].generatetoaddress(1, self.address, 1), 0) + self.rollback_to(block_height) + + def test_invalid_utf8_encoding_token_name_dst20_migration_tx(self): + block_height = self.nodes[0].getblockcount() + + self.node.createtoken( + { + "symbol": "InvalidUTF8TokenName", + "name": "InvalidUTF8TokenNameIsThisOne🤩", + "isDAT": True, + "collateralAddress": self.address, + } + ) + self.nodes[0].generate(2) + + # enable EVM, transferdomain, DVM to EVM transfers and EVM to DVM transfers + self.nodes[0].setgov( + { + "ATTRIBUTES": { + "v0/params/feature/evm": "true", + } + } + ) + self.nodes[0].generate(1) + + # Trigger EVM genesis DST20 migration + assert_equal(self.nodes[0].generatetoaddress(1, self.address, 1), 0) + self.rollback_to(block_height) + def test_dst20_migration_txs(self): block_height = self.nodes[0].getblockcount() @@ -73,7 +128,16 @@ def test_dst20_migration_txs(self): "collateralAddress": self.address, } ) - self.nodes[0].generate(1) + # create DST20 token with maximum byte size limit + self.node.createtoken( + { + "symbol": "Test", + "name": "TheTokenWithNameMax30ByteLimit", # 30 bytes + "isDAT": True, + "collateralAddress": self.address, + } + ) + self.nodes[0].generate(2) # enable EVM, transferdomain, DVM to EVM transfers and EVM to DVM transfers self.nodes[0].setgov( @@ -981,9 +1045,13 @@ def run_test(self): # Generate chain self.node.generate(150) self.nodes[0].utxostoaccount({self.address: "1000@DFI"}) + + # pre-metachain fork height self.nodes[0].generate(1) # Create token and check DST20 migration pre EVM activation + self.test_invalid_too_long_token_name_dst20_migration_tx() + self.test_invalid_utf8_encoding_token_name_dst20_migration_tx() self.test_dst20_migration_txs() # Create token before EVM diff --git a/test/functional/feature_evm_eip1559_fees.py b/test/functional/feature_evm_eip1559_fees.py index b78b94dfa1f..a722c0a04a4 100644 --- a/test/functional/feature_evm_eip1559_fees.py +++ b/test/functional/feature_evm_eip1559_fees.py @@ -37,6 +37,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ] ] diff --git a/test/functional/feature_evm_fee.py b/test/functional/feature_evm_fee.py index 6d938ffbb1e..43459d98fe1 100755 --- a/test/functional/feature_evm_fee.py +++ b/test/functional/feature_evm_fee.py @@ -33,6 +33,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_gas.py b/test/functional/feature_evm_gas.py index 2c3bf370c4f..41abef02399 100644 --- a/test/functional/feature_evm_gas.py +++ b/test/functional/feature_evm_gas.py @@ -35,6 +35,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_genesis.py b/test/functional/feature_evm_genesis.py index 947c2c2e07f..9460159018d 100755 --- a/test/functional/feature_evm_genesis.py +++ b/test/functional/feature_evm_genesis.py @@ -37,6 +37,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] @@ -140,6 +141,7 @@ def test_start_state_from_json(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_logs.py b/test/functional/feature_evm_logs.py index 373a28197ab..9ea4362364d 100755 --- a/test/functional/feature_evm_logs.py +++ b/test/functional/feature_evm_logs.py @@ -36,6 +36,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_mempool.py b/test/functional/feature_evm_mempool.py index 67e5750a936..a19772b167b 100755 --- a/test/functional/feature_evm_mempool.py +++ b/test/functional/feature_evm_mempool.py @@ -34,6 +34,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", "-ethdebug=1", ], diff --git a/test/functional/feature_evm_miner.py b/test/functional/feature_evm_miner.py index af3e92a7d53..e8974df35f1 100755 --- a/test/functional/feature_evm_miner.py +++ b/test/functional/feature_evm_miner.py @@ -37,6 +37,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_proxy.py b/test/functional/feature_evm_proxy.py index fb3d0060824..0ab19414eb5 100755 --- a/test/functional/feature_evm_proxy.py +++ b/test/functional/feature_evm_proxy.py @@ -33,6 +33,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_rollback.py b/test/functional/feature_evm_rollback.py index 3577f62ecf4..6e7a6b39387 100755 --- a/test/functional/feature_evm_rollback.py +++ b/test/functional/feature_evm_rollback.py @@ -30,6 +30,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", "-ethdebug=1", ], diff --git a/test/functional/feature_evm_rpc.py b/test/functional/feature_evm_rpc.py index 2d588cfe47d..0e408366c71 100755 --- a/test/functional/feature_evm_rpc.py +++ b/test/functional/feature_evm_rpc.py @@ -39,6 +39,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", "-ethdebug=1", ], diff --git a/test/functional/feature_evm_rpc_fee_history.py b/test/functional/feature_evm_rpc_fee_history.py index 3bd9f420963..e119a273be3 100755 --- a/test/functional/feature_evm_rpc_fee_history.py +++ b/test/functional/feature_evm_rpc_fee_history.py @@ -41,6 +41,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_rpc_filters.py b/test/functional/feature_evm_rpc_filters.py index e02438c6218..d9889ec14ce 100755 --- a/test/functional/feature_evm_rpc_filters.py +++ b/test/functional/feature_evm_rpc_filters.py @@ -36,6 +36,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_rpc_transaction.py b/test/functional/feature_evm_rpc_transaction.py index f73b2e288fa..66acabd0649 100755 --- a/test/functional/feature_evm_rpc_transaction.py +++ b/test/functional/feature_evm_rpc_transaction.py @@ -44,6 +44,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", "-ethdebug=1", ], diff --git a/test/functional/feature_evm_smart_contract.py b/test/functional/feature_evm_smart_contract.py index 2e7ac9fa948..269a209fd23 100755 --- a/test/functional/feature_evm_smart_contract.py +++ b/test/functional/feature_evm_smart_contract.py @@ -32,6 +32,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_transaction_replacement.py b/test/functional/feature_evm_transaction_replacement.py index 3d2b731711d..5ad7bf7f0f4 100755 --- a/test/functional/feature_evm_transaction_replacement.py +++ b/test/functional/feature_evm_transaction_replacement.py @@ -40,6 +40,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/feature_evm_transferdomain.py b/test/functional/feature_evm_transferdomain.py index 4bd2a992836..3aa33383dd0 100755 --- a/test/functional/feature_evm_transferdomain.py +++ b/test/functional/feature_evm_transferdomain.py @@ -56,6 +56,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=150", + "-df23upgradeheight=105", "-subsidytest=1", ] self.extra_args = [node_args, node_args] diff --git a/test/functional/feature_evm_vmmap_rpc.py b/test/functional/feature_evm_vmmap_rpc.py index 637fd89b2de..c322b065a58 100755 --- a/test/functional/feature_evm_vmmap_rpc.py +++ b/test/functional/feature_evm_vmmap_rpc.py @@ -39,6 +39,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ] self.extra_args = [extra_args, extra_args] diff --git a/test/functional/feature_icx_orderbook.py b/test/functional/feature_icx_orderbook.py index c555905f4e4..48e231f57f0 100755 --- a/test/functional/feature_icx_orderbook.py +++ b/test/functional/feature_icx_orderbook.py @@ -33,6 +33,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=3505", + "-df23upgradeheight=3505", "-txindex=1", ], [ @@ -51,6 +52,7 @@ def set_test_params(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=3505", + "-df23upgradeheight=3505", "-txindex=1", ], ] diff --git a/test/functional/feature_on_chain_government_voting_scenarios.py b/test/functional/feature_on_chain_government_voting_scenarios.py index 473f6f8c02b..0c6780b46d4 100755 --- a/test/functional/feature_on_chain_government_voting_scenarios.py +++ b/test/functional/feature_on_chain_government_voting_scenarios.py @@ -34,6 +34,7 @@ def set_test_params(self): "-fortcanninggreatworldheight=94", "-grandcentralheight=101", f"-metachainheight={NEXT_NETWORK_UPGRADE_HEIGHT}", + f"-df23upgradeheight={NEXT_NETWORK_UPGRADE_HEIGHT}", "-rpc-governance-accept-neutral=1", "-simulatemainnet=1", ], diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 4e7aea47d54..30d4956efb4 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -71,6 +71,7 @@ def run_test(self): "-fortcanningepilogueheight=96", "-grandcentralheight=101", "-metachainheight=105", + "-df23upgradeheight=105", "-subsidytest=1", ], ] diff --git a/test/functional/rpc_updatemasternode.py b/test/functional/rpc_updatemasternode.py index ea224faf36b..17a176cfa3d 100755 --- a/test/functional/rpc_updatemasternode.py +++ b/test/functional/rpc_updatemasternode.py @@ -359,6 +359,7 @@ def run_test(self): "-bayfrontheight=50", "-grandcentralheight=1", "-metachainheight=510", + "-df23upgradeheight=510", ], )