From 1943c1b710cce0d655956b4dd404d0121d81c96c Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 20 Mar 2024 12:02:48 +0100 Subject: [PATCH] capi: split `silkworm_execute_blocks()` into functions with ext and int txns (#1917) --- cmd/capi/execute.cpp | 41 ++-- silkworm/capi/silkworm.cpp | 346 ++++++++++++++++++++------------ silkworm/capi/silkworm.h | 36 +++- silkworm/capi/silkworm_test.cpp | 293 ++++++++++++++++++++++++--- 4 files changed, 538 insertions(+), 178 deletions(-) diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp index 01487631e3..00e2336d06 100644 --- a/cmd/capi/execute.cpp +++ b/cmd/capi/execute.cpp @@ -221,25 +221,6 @@ std::vector collect_all_snapshots(const SnapshotRepositor return snapshot_sequence; } -std::tuple execute(SilkwormHandle handle, ExecuteBlocksSettings settings, ::mdbx::env env, MDBX_txn* txn, ChainId chain_id, BlockNum start_block) { - const auto max_block{settings.max_block}; - const auto batch_size{settings.batch_size}; - const auto write_change_sets{settings.write_change_sets}; - const auto write_receipts{settings.write_receipts}; - const auto write_call_traces{settings.write_call_traces}; - BlockNum last_executed_block{0}; - int mdbx_error_code{0}; - const uint64_t count{max_block - start_block + 1}; - SILK_DEBUG << "Execute blocks start_block=" << start_block << " end_block=" << max_block << " count=" << count << " batch_size=" << batch_size << " start"; - const int status_code = silkworm_execute_blocks( - handle, env, txn, chain_id, - start_block, max_block, batch_size, - write_change_sets, write_receipts, write_call_traces, - &last_executed_block, &mdbx_error_code); - SILK_DEBUG << "Execute blocks start_block=" << start_block << " end_block=" << max_block << " count=" << count << " batch_size=" << batch_size << " done"; - return {status_code, last_executed_block, mdbx_error_code}; -} - int execute_with_internal_txn(SilkwormHandle handle, ExecuteBlocksSettings settings, ::mdbx::env& env) { db::ROTxnManaged ro_txn{env}; const auto chain_config{db::read_chain_config(ro_txn)}; @@ -247,8 +228,17 @@ int execute_with_internal_txn(SilkwormHandle handle, ExecuteBlocksSettings setti const auto chain_id{chain_config->chain_id}; ro_txn.abort(); - const auto start_block{settings.start_block}; - const auto [status_code, last_executed_block, mdbx_error_code] = execute(handle, settings, env, nullptr, chain_id, start_block); + BlockNum last_executed_block{0}; + int mdbx_error_code{0}; + const uint64_t count{settings.max_block - settings.start_block + 1}; + SILK_DEBUG << "Execute blocks start_block=" << settings.start_block << " end_block=" << settings.max_block << " count=" << count << " batch_size=" << settings.batch_size << " start"; + const int status_code = silkworm_execute_blocks_perpetual( + handle, env, chain_id, + settings.start_block, settings.max_block, settings.batch_size, + settings.write_change_sets, settings.write_receipts, settings.write_call_traces, + &last_executed_block, &mdbx_error_code); + SILK_DEBUG << "Execute blocks start_block=" << settings.start_block << " end_block=" << settings.max_block << " count=" << count << " batch_size=" << settings.batch_size << " done"; + if (status_code != SILKWORM_OK) { SILK_ERROR << "execute_with_internal_txn failed [code=" << std::to_string(status_code) << (status_code == SILKWORM_MDBX_ERROR ? " mdbx_error_code=" + std::to_string(mdbx_error_code) : "") @@ -270,7 +260,14 @@ int execute_with_external_txn(SilkwormHandle handle, ExecuteBlocksSettings setti auto start_block{settings.start_block}; const auto max_block{settings.max_block}; while (start_block <= max_block) { - const auto [status_code, last_executed_block, mdbx_error_code] = execute(handle, settings, env, *rw_txn, chain_id, start_block); + BlockNum last_executed_block{0}; + int mdbx_error_code{0}; + const int status_code = silkworm_execute_blocks_ephemeral( + handle, *rw_txn, chain_id, + settings.start_block, settings.max_block, settings.batch_size, + settings.write_change_sets, settings.write_receipts, settings.write_call_traces, + &last_executed_block, &mdbx_error_code); + if (status_code != SILKWORM_OK) { SILK_ERROR << "execute_with_external_txn failed [code=" << std::to_string(status_code) << (status_code == SILKWORM_MDBX_ERROR ? " mdbx_error_code=" + std::to_string(mdbx_error_code) : "") diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 8f9f0bc255..13c24484ed 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -58,6 +58,8 @@ static log::Settings kLogSettingsLikeErigon{ .log_trim = true, // compact rendering (i.e. no whitespaces) }; static constexpr size_t kMaxBlockBufferSize{100}; +static constexpr size_t kAnalysisCacheSize{5'000}; +static constexpr size_t kMaxPrefetchedBlocks{10'240}; using SteadyTimePoint = std::chrono::time_point; @@ -68,7 +70,7 @@ struct ExecutionProgress { size_t processed_blocks{0}; size_t processed_transactions{0}; size_t processed_gas{0}; - float gas_state_perc{0.0}; + float batch_progress_perc{0.0}; }; //! Kind of match to perform between Erigon and Silkworm libmdbx versions @@ -162,6 +164,8 @@ static log::Args log_args_for_exec_progress(ExecutionProgress& progress, uint64_ progress.processed_blocks = 0; progress.processed_transactions = 0; progress.processed_gas = 0; + std::stringstream batch_progress_perc; + batch_progress_perc << std::fixed << std::setprecision(2) << progress.batch_progress_perc * 100 << "%"; return { "number", std::to_string(current_block), @@ -171,8 +175,8 @@ static log::Args log_args_for_exec_progress(ExecutionProgress& progress, uint64_ float_to_string(speed_transactions), "Mgas/s", float_to_string(speed_mgas), - "gasState", - float_to_string(progress.gas_state_perc)}; + "batchProgress", + batch_progress_perc.str()}; } //! A signal handler guard using RAII pattern to acquire/release signal handling @@ -473,16 +477,98 @@ class BlockProvider { BlockNum max_block_; }; +class BlockExecutor { + public: + BlockExecutor(const ChainConfig* chain_config, bool write_receipts, bool write_call_traces, bool write_change_sets, size_t max_batch_size) + : chain_config_{chain_config}, + protocol_rule_set_{protocol::rule_set_factory(*chain_config_)}, + write_receipts_{write_receipts}, + write_call_traces_{write_call_traces}, + write_change_sets_{write_change_sets}, + analysis_cache_{kAnalysisCacheSize}, + state_pool_{}, + progress_{.start_time = std::chrono::steady_clock::now()}, + log_time_{progress_.start_time + 20s}, + max_batch_size_{max_batch_size} {} + + silkworm::ValidationResult execute_single(const silkworm::Block& block, silkworm::db::Buffer& state_buffer) { + ExecutionProcessor processor{block, *protocol_rule_set_, state_buffer, *chain_config_}; + processor.evm().analysis_cache = &analysis_cache_; + processor.evm().state_pool = &state_pool_; + + CallTraces traces; + CallTracer tracer{traces}; + if (write_call_traces_) { + processor.evm().add_tracer(tracer); + } + + std::vector receipts; + const auto result{processor.execute_and_write_block(receipts)}; + + if (result != ValidationResult::kOk) { + return result; + } + + if (write_receipts_) { + state_buffer.insert_receipts(block.header.number, receipts); + } + if (write_call_traces_) { + state_buffer.insert_call_traces(block.header.number, traces); + } + + state_buffer.write_history_to_db(write_change_sets_); + + progress_.processed_blocks++; + progress_.processed_transactions += block.transactions.size(); + progress_.processed_gas += block.header.gas_used; + + const auto now{std::chrono::steady_clock::now()}; + if (log_time_ <= now) { + progress_.batch_progress_perc = float(state_buffer.current_batch_state_size()) / float(max_batch_size_); + progress_.end_time = now; + log::Info{"[4/12 Execution] Executed blocks", // NOLINT(*-unused-raii) + log_args_for_exec_progress(progress_, block.header.number)}; + log_time_ = now + 20s; + } + + return result; + } + + private: + const ChainConfig* chain_config_; + silkworm::protocol::RuleSetPtr protocol_rule_set_; + bool write_receipts_; + bool write_call_traces_; + bool write_change_sets_; + AnalysisCache analysis_cache_; + ObjectPool state_pool_; + ExecutionProgress progress_; + SteadyTimePoint log_time_; + const size_t max_batch_size_; +}; + +inline bool signal_check(SteadyTimePoint& signal_check_time) { + const auto now{std::chrono::steady_clock::now()}; + if (signal_check_time <= now) { + if (SignalHandler::signalled()) { + return true; + } + signal_check_time += 5s; + } + + return false; +} + SILKWORM_EXPORT -int silkworm_execute_blocks(SilkwormHandle handle, MDBX_env* mdbx_env, MDBX_txn* mdbx_txn, uint64_t chain_id, - uint64_t start_block, uint64_t max_block, uint64_t batch_size, - bool write_change_sets, bool write_receipts, bool write_call_traces, - uint64_t* last_executed_block, int* mdbx_error_code) SILKWORM_NOEXCEPT { +int silkworm_execute_blocks_ephemeral(SilkwormHandle handle, MDBX_txn* mdbx_txn, uint64_t chain_id, + uint64_t start_block, uint64_t max_block, uint64_t batch_size, + bool write_change_sets, bool write_receipts, bool write_call_traces, + uint64_t* last_executed_block, int* mdbx_error_code) SILKWORM_NOEXCEPT { if (!handle) { return SILKWORM_INVALID_HANDLE; } - if (!mdbx_env) { - return SILKWORM_INVALID_MDBX_ENV; + if (!mdbx_txn) { + return SILKWORM_INVALID_MDBX_TXN; } if (start_block > max_block) { return SILKWORM_INVALID_BLOCK_RANGE; @@ -491,153 +577,157 @@ int silkworm_execute_blocks(SilkwormHandle handle, MDBX_env* mdbx_env, MDBX_txn* if (!chain_info) { return SILKWORM_UNKNOWN_CHAIN_ID; } - const ChainConfig* chain_config{*chain_info}; - const bool use_external_txn{mdbx_txn != nullptr}; - - // Wrap MDBX env into an internal *unmanaged* env, i.e. MDBX env is only used but its lifecycle is untouched - db::EnvUnmanaged unmanaged_env{mdbx_env}; - SILK_TRACE << "[Silkworm Exec] env=" << unmanaged_env.get_path().string() << " external_txn=" << std::boolalpha << use_external_txn; - SignalHandlerGuard signal_guard; - try { - std::unique_ptr txn; - if (use_external_txn) { - // Wrap MDBX txn into an internal *unmanaged* txn, i.e. MDBX txn is only used but neither committed nor aborted - txn = std::make_unique(mdbx_txn); - } else { - // Create a *managed* MDBX txn, i.e. MDBX txn is used and then either committed or aborted - txn = std::make_unique(unmanaged_env); - } - - const auto db_path{txn->db().get_path()}; - db::Buffer state_buffer{*txn, /*prune_history_threshold=*/0}; - BoundedBuffer> block_buffer{kMaxBlockBufferSize}; - BlockProvider block_provider{&block_buffer, txn->db(), start_block, max_block}; - boost::strict_scoped_thread block_provider_thread(block_provider); + try { + auto txn = db::RWTxnUnmanaged{mdbx_txn}; + const auto db_path{txn.db().get_path()}; - static constexpr size_t kCacheSize{5'000}; - AnalysisCache analysis_cache{kCacheSize}; - ObjectPool state_pool; + db::Buffer state_buffer{txn, /*prune_history_threshold=*/0}; const size_t max_batch_size{batch_size}; + auto signal_check_time{std::chrono::steady_clock::now()}; + + BlockNum block_number{start_block}; + db::DataModel da_layer{txn}; + BlockExecutor block_executor{*chain_info, write_receipts, write_call_traces, write_change_sets, max_batch_size}; + boost::circular_buffer prefetched_blocks{/*buffer_capacity=*/kMaxPrefetchedBlocks}; + + while (block_number <= max_block) { + while (state_buffer.current_batch_state_size() < max_batch_size && block_number <= max_block) { + if (prefetched_blocks.empty()) { + const auto num_blocks{std::min(size_t(max_block - block_number + 1), kMaxPrefetchedBlocks)}; + SILK_TRACE << "Prefetching " << num_blocks << " blocks start"; + for (BlockNum n{block_number}; n < block_number + num_blocks; ++n) { + prefetched_blocks.push_back(); + const bool success{da_layer.read_block(n, /*read_senders=*/true, prefetched_blocks.back())}; + if (!success) { + return SILKWORM_BLOCK_NOT_FOUND; + } + } + SILK_TRACE << "Prefetching " << num_blocks << " blocks done"; + } + const Block& block{prefetched_blocks.front()}; - // Transform batch size limit into gas units (Ggas = Giga gas) - const size_t gas_max_batch_size{batch_size * 2_Kibi}; // 256MB -> 512Ggas roughly - - ExecutionProgress progress{.start_time = std::chrono::steady_clock::now()}; - auto signal_check_time{progress.start_time}; - auto log_time{progress.start_time}; - - size_t gas_batch_size{0}; - - const auto protocol_rule_set{protocol::rule_set_factory(*chain_config)}; - if (!protocol_rule_set) { - return SILKWORM_UNKNOWN_CHAIN_ID; - } + const auto result{block_executor.execute_single(block, state_buffer)}; - std::optional block; - db::DataModel da_layer{*txn}; - Block b; - for (BlockNum block_number{start_block}; block_number <= max_block; ++block_number) { - if (use_external_txn) { - if (const bool ok{da_layer.read_block(block_number, /*read_senders=*/true, b)}; ok) { - block = std::move(b); + if (result != ValidationResult::kOk) { + return SILKWORM_INVALID_BLOCK; + } + if (signal_check(signal_check_time)) { + return SILKWORM_TERMINATION_SIGNAL; } - } else { - block_buffer.pop_back(&block); - } - if (!block || !block.has_value()) { - block_buffer.terminate_and_release_all(); - return SILKWORM_BLOCK_NOT_FOUND; - } - SILKWORM_ASSERT(block->header.number == block_number); - - ExecutionProcessor processor{*block, *protocol_rule_set, state_buffer, *chain_config}; - processor.evm().analysis_cache = &analysis_cache; - processor.evm().state_pool = &state_pool; - CallTraces traces; - CallTracer tracer{traces}; - if (write_call_traces) { - processor.evm().add_tracer(tracer); - } - std::vector receipts; - const auto result{processor.execute_and_write_block(receipts)}; - if (result != ValidationResult::kOk) { - block_buffer.terminate_and_release_all(); - return SILKWORM_INVALID_BLOCK; + ++block_number; + prefetched_blocks.pop_front(); } - if (write_receipts) { - state_buffer.insert_receipts(block->header.number, receipts); - } - if (write_call_traces) { - state_buffer.insert_call_traces(block->header.number, traces); - } + auto last_block_number = block_number - 1; + log::Info{"[4/12 Execution] Flushing state", // NOLINT(*-unused-raii) + log_args_for_exec_flush(state_buffer, max_batch_size, last_block_number)}; + state_buffer.write_state_to_db(); + // Always save the Execution stage progress when state batch is flushed + db::stages::write_stage_progress(txn, db::stages::kExecutionKey, last_block_number); if (last_executed_block) { - *last_executed_block = block->header.number; + *last_executed_block = last_block_number; } + } + return SILKWORM_OK; + } catch (const mdbx::exception& e) { + if (mdbx_error_code) { + *mdbx_error_code = e.error().code(); + } + return SILKWORM_MDBX_ERROR; + } catch (const DecodingError&) { + return SILKWORM_DECODING_ERROR; + } catch (const std::exception& e) { + SILK_ERROR << "exception: " << e.what(); + return SILKWORM_INTERNAL_ERROR; + } catch (...) { + SILK_ERROR << "unknown exception"; + return SILKWORM_UNKNOWN_ERROR; + } +} + +SILKWORM_EXPORT +int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, uint64_t chain_id, + uint64_t start_block, uint64_t max_block, uint64_t batch_size, + bool write_change_sets, bool write_receipts, bool write_call_traces, + uint64_t* last_executed_block, int* mdbx_error_code) SILKWORM_NOEXCEPT { + if (!handle) { + return SILKWORM_INVALID_HANDLE; + } + if (!mdbx_env) { + return SILKWORM_INVALID_MDBX_ENV; + } + if (start_block > max_block) { + return SILKWORM_INVALID_BLOCK_RANGE; + } + const auto chain_info = kKnownChainConfigs.find(chain_id); + if (!chain_info) { + return SILKWORM_UNKNOWN_CHAIN_ID; + } + SignalHandlerGuard signal_guard; - ++progress.processed_blocks; - progress.processed_transactions += block->transactions.size(); - progress.processed_gas += block->header.gas_used; - gas_batch_size += block->header.gas_used; - - // Always flush history for single processed block (no batching) - state_buffer.write_history_to_db(write_change_sets); - - // Flush state buffer if we've reached the target batch size - if (state_buffer.current_batch_state_size() >= max_batch_size) { - log::Info{"[4/12 Execution] Flushing state", // NOLINT(*-unused-raii) - log_args_for_exec_flush(state_buffer, max_batch_size, block->header.number)}; - state_buffer.write_state_to_db(); - // Always save the Execution stage progess when state batch is flushed - db::stages::write_stage_progress(*txn, db::stages::kExecutionKey, block->header.number); - gas_batch_size = 0; - // Commit and renew only in case of internally managed transaction - if (!use_external_txn) { - StopWatch sw{/*auto_start=*/true}; - txn->commit_and_renew(); - const auto [elapsed, _]{sw.stop()}; - log::Info("[4/12 Execution] Commit state+history", // NOLINT(*-unused-raii) - log_args_for_exec_commit(sw.since_start(elapsed), db_path)); + try { + // Wrap MDBX env into an internal *unmanaged* env, i.e. MDBX env is only used but its lifecycle is untouched + db::EnvUnmanaged unmanaged_env{mdbx_env}; + auto txn = db::RWTxnManaged{unmanaged_env}; + const auto db_path{unmanaged_env.get_path()}; + + db::Buffer state_buffer{txn, /*prune_history_threshold=*/0}; + BoundedBuffer> block_buffer{kMaxBlockBufferSize}; + BlockProvider block_provider{&block_buffer, unmanaged_env, start_block, max_block}; + boost::strict_scoped_thread block_provider_thread(block_provider); + + const size_t max_batch_size{batch_size}; + auto signal_check_time{std::chrono::steady_clock::now()}; + + std::optional block; + BlockNum block_number{start_block}; + BlockExecutor block_executor{*chain_info, write_receipts, write_call_traces, write_change_sets, max_batch_size}; + + while (block_number <= max_block) { + while (state_buffer.current_batch_state_size() < max_batch_size && block_number <= max_block) { + block_buffer.pop_back(&block); + if (!block) { + block_buffer.terminate_and_release_all(); + return SILKWORM_BLOCK_NOT_FOUND; } - } + SILKWORM_ASSERT(block->header.number == block_number); - const auto now{std::chrono::steady_clock::now()}; - if (signal_check_time <= now) { - if (SignalHandler::signalled()) { + const auto result{block_executor.execute_single(*block, state_buffer)}; + + if (result != ValidationResult::kOk) { + block_buffer.terminate_and_release_all(); + return SILKWORM_INVALID_BLOCK; + } + if (signal_check(signal_check_time)) { block_buffer.terminate_and_release_all(); return SILKWORM_TERMINATION_SIGNAL; } - signal_check_time = now + 5s; - } - if (log_time <= now) { - progress.gas_state_perc = float(gas_batch_size) / float(gas_max_batch_size); - progress.end_time = now; - log::Info{"[4/12 Execution] Executed blocks", // NOLINT(*-unused-raii) - log_args_for_exec_progress(progress, block->header.number)}; - log_time = now + 20s; + + ++block_number; } - } - log::Info{"[4/12 Execution] Flushing state", // NOLINT(*-unused-raii) - log_args_for_exec_flush(state_buffer, max_batch_size, max_block)}; - state_buffer.write_state_to_db(); - // Always save the Execution stage progess when last state batch is flushed - db::stages::write_stage_progress(*txn, db::stages::kExecutionKey, max_block); - // Commit only in case of internally managed transaction - if (!use_external_txn) { StopWatch sw{/*auto_start=*/true}; - txn->commit_and_stop(); + log::Info{"[4/12 Execution] Flushing state", // NOLINT(*-unused-raii) + log_args_for_exec_flush(state_buffer, max_batch_size, block->header.number)}; + state_buffer.write_state_to_db(); + // Always save the Execution stage progress when state batch is flushed + db::stages::write_stage_progress(txn, db::stages::kExecutionKey, block->header.number); + // Commit and renew only in case of internally managed transaction + txn.commit_and_renew(); const auto [elapsed, _]{sw.stop()}; log::Info("[4/12 Execution] Commit state+history", // NOLINT(*-unused-raii) log_args_for_exec_commit(sw.since_start(elapsed), db_path)); + + if (last_executed_block) { + *last_executed_block = block->header.number; + } } return SILKWORM_OK; - } catch (const mdbx::exception& e) { if (mdbx_error_code) { *mdbx_error_code = e.error().code(); diff --git a/silkworm/capi/silkworm.h b/silkworm/capi/silkworm.h index 399ffcca3f..f40eda52b4 100644 --- a/silkworm/capi/silkworm.h +++ b/silkworm/capi/silkworm.h @@ -59,6 +59,7 @@ extern "C" { #define SILKWORM_TERMINATION_SIGNAL 15 #define SILKWORM_SERVICE_ALREADY_STARTED 16 #define SILKWORM_INCOMPATIBLE_LIBMDBX 17 +#define SILKWORM_INVALID_MDBX_TXN 18 typedef struct MDBX_env MDBX_env; typedef struct MDBX_txn MDBX_txn; @@ -187,9 +188,8 @@ SILKWORM_EXPORT int silkworm_sentry_start(SilkwormHandle handle, const struct Si SILKWORM_EXPORT int silkworm_sentry_stop(SilkwormHandle handle) SILKWORM_NOEXCEPT; /** - * \brief Execute a batch of blocks and write resulting changes into the database. + * \brief Execute a batch of blocks and push changes to the given database transaction. No data is commited. * \param[in] handle A valid Silkworm instance handle, got with silkworm_init. - * \param[in] env An valid MDBX environment. Must not be zero. * \param[in] txn A valid external read-write MDBX transaction or zero if an internal one must be used. * This function does not commit nor abort the transaction. * \param[in] chain_id EIP-155 chain ID. SILKWORM_UNKNOWN_CHAIN_ID is returned in case of an unknown or unsupported chain. @@ -209,11 +209,39 @@ SILKWORM_EXPORT int silkworm_sentry_stop(SilkwormHandle handle) SILKWORM_NOEXCEP * SILKWORM_BLOCK_NOT_FOUND is probably OK: it simply means that the execution reached the end of the chain * (blocks up to and incl. last_executed_block were still executed). */ -SILKWORM_EXPORT int silkworm_execute_blocks( - SilkwormHandle handle, MDBX_env* env, MDBX_txn* txn, uint64_t chain_id, uint64_t start_block, uint64_t max_block, +SILKWORM_EXPORT int silkworm_execute_blocks_ephemeral( + SilkwormHandle handle, MDBX_txn* txn, uint64_t chain_id, uint64_t start_block, uint64_t max_block, uint64_t batch_size, bool write_change_sets, bool write_receipts, bool write_call_traces, uint64_t* last_executed_block, int* mdbx_error_code) SILKWORM_NOEXCEPT; + +/** + * \brief Execute a batch of blocks and write resulting changes into the database. + * \param[in] handle A valid Silkworm instance handle, got with silkworm_init. + * \param[in] mdbx_env An valid MDBX environment. Must not be zero. + * \param[in] chain_id EIP-155 chain ID. SILKWORM_UNKNOWN_CHAIN_ID is returned in case of an unknown or unsupported chain. + * \param[in] start_block The block height to start the execution from. + * \param[in] max_block Do not execute after this block. + * max_block may be executed, or the execution may stop earlier if the batch is full. + * \param[in] batch_size The size of DB changes to accumulate before returning from this method. + * Pass 0 if you want to execute just 1 block. + * \param[in] write_change_sets Whether to write state changes into the DB. + * \param[in] write_receipts Whether to write CBOR-encoded receipts into the DB. + * \param[in] write_call_traces Whether to write call traces into the DB. + * \param[out] last_executed_block The height of the last successfully executed block. + * Not written to if no blocks were executed, otherwise *last_executed_block ≤ max_block. + * \param[out] mdbx_error_code If an MDBX error occurs (this function returns kSilkwormMdbxError) + * and mdbx_error_code isn't NULL, it's populated with the relevant MDBX error code. + * \return SILKWORM_OK (=0) on success, a non-zero error value on failure. + * SILKWORM_BLOCK_NOT_FOUND is probably OK: it simply means that the execution reached the end of the chain + * (blocks up to and incl. last_executed_block were still executed). + */ +SILKWORM_EXPORT int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, uint64_t chain_id, + uint64_t start_block, uint64_t max_block, uint64_t batch_size, + bool write_change_sets, bool write_receipts, bool write_call_traces, + uint64_t* last_executed_block, int* mdbx_error_code) SILKWORM_NOEXCEPT; + + /** * \brief Finalize the Silkworm C API library. * \param[in] handle A valid Silkworm instance handle got with silkworm_init. diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index 1c7cdf4c01..f10a826d10 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -141,8 +141,7 @@ struct SilkwormLibrary { int mdbx_error_code{0}; }; - ExecutionResult execute_blocks(MDBX_env* env, - MDBX_txn* txn, + ExecutionResult execute_blocks(MDBX_txn* txn, uint64_t chain_id, uint64_t start_block, uint64_t max_block, @@ -152,10 +151,27 @@ struct SilkwormLibrary { bool write_call_traces) { ExecutionResult result; result.execute_block_result = - silkworm_execute_blocks(handle_, env, txn, - chain_id, start_block, max_block, batch_size, - write_change_sets, write_receipts, write_call_traces, - &result.last_executed_block, &result.mdbx_error_code); + silkworm_execute_blocks_ephemeral(handle_, txn, + chain_id, start_block, max_block, batch_size, + write_change_sets, write_receipts, write_call_traces, + &result.last_executed_block, &result.mdbx_error_code); + return result; + } + + ExecutionResult execute_blocks_perpetual(MDBX_env* env, + uint64_t chain_id, + uint64_t start_block, + uint64_t max_block, + uint64_t batch_size, + bool write_change_sets, + bool write_receipts, + bool write_call_traces) { + ExecutionResult result; + result.execute_block_result = + silkworm_execute_blocks_perpetual(handle_, env, + chain_id, start_block, max_block, batch_size, + write_change_sets, write_receipts, write_call_traces, + &result.last_executed_block, &result.mdbx_error_code); return result; } @@ -167,7 +183,7 @@ struct SilkwormLibrary { SilkwormHandle handle_{nullptr}; }; -TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks: block not found", "[silkworm][capi]") { +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_ephemeral: block not found", "[silkworm][capi]") { // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII SilkwormLibrary silkworm_lib{db.get_path()}; @@ -175,14 +191,66 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks: block not found", "[si const uint64_t batch_size{256 * kMebi}; BlockNum start_block{10}; // This does not exist, TestDatabaseContext db contains up to block 9 BlockNum end_block{100}; + db::RWTxnManaged external_txn{db}; const auto result0{ - silkworm_lib.execute_blocks(db, nullptr, chain_id, start_block, end_block, batch_size, + silkworm_lib.execute_blocks(*external_txn, chain_id, start_block, end_block, batch_size, true, true, true)}; + CHECK_NOTHROW(external_txn.commit_and_stop()); CHECK(result0.execute_block_result == SILKWORM_BLOCK_NOT_FOUND); CHECK(result0.last_executed_block == 0); CHECK(result0.mdbx_error_code == 0); } +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_perpetual: block not found", "[silkworm][capi]") { + // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII + SilkwormLibrary silkworm_lib{db.get_path()}; + + const int chain_id{1}; + const uint64_t batch_size{256 * kMebi}; + BlockNum start_block{10}; // This does not exist, TestDatabaseContext db contains up to block 9 + BlockNum end_block{100}; + const auto result0{ + silkworm_lib.execute_blocks_perpetual(db, chain_id, start_block, end_block, batch_size, + true, true, true)}; + CHECK(result0.execute_block_result == SILKWORM_BLOCK_NOT_FOUND); + CHECK(result0.last_executed_block == 0); + CHECK(result0.mdbx_error_code == 0); +} + +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_ephemeral: chain id not found", "[silkworm][capi]") { + // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII + SilkwormLibrary silkworm_lib{db.get_path()}; + + const uint64_t chain_id{1000000}; + const uint64_t batch_size{256 * kMebi}; + BlockNum start_block{1}; + BlockNum end_block{2}; + db::RWTxnManaged external_txn{db}; + const auto result0{ + silkworm_lib.execute_blocks(*external_txn, chain_id, start_block, end_block, batch_size, + true, true, true)}; + CHECK_NOTHROW(external_txn.commit_and_stop()); + CHECK(result0.execute_block_result == SILKWORM_UNKNOWN_CHAIN_ID); + CHECK(result0.last_executed_block == 0); + CHECK(result0.mdbx_error_code == 0); +} + +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_perpetual: chain id not found", "[silkworm][capi]") { + // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII + SilkwormLibrary silkworm_lib{db.get_path()}; + + const uint64_t chain_id{1000000}; + const uint64_t batch_size{256 * kMebi}; + BlockNum start_block{1}; + BlockNum end_block{2}; + const auto result0{ + silkworm_lib.execute_blocks_perpetual(db, chain_id, start_block, end_block, batch_size, + true, true, true)}; + CHECK(result0.execute_block_result == SILKWORM_UNKNOWN_CHAIN_ID); + CHECK(result0.last_executed_block == 0); + CHECK(result0.mdbx_error_code == 0); +} + static void insert_block(mdbx::env& env, Block& block) { auto block_hash = block.header.hash(); auto block_hash_key = db::block_key(block.header.number, block_hash.bytes); @@ -206,7 +274,7 @@ static void insert_block(mdbx::env& env, Block& block) { rw_txn.commit_and_stop(); } -TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks single block: OK", "[silkworm][capi]") { +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_ephemeral single block: OK", "[silkworm][capi]") { // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII SilkwormLibrary silkworm_lib{db.get_path()}; @@ -217,8 +285,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks single block: OK", "[si const bool write_call_traces{false}; // For coherence but don't care auto execute_blocks = [&](auto tx, auto start_block, auto end_block) { - return silkworm_lib.execute_blocks(db, - tx, + return silkworm_lib.execute_blocks(tx, chain_id, start_block, end_block, @@ -258,9 +325,11 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks single block: OK", "[si insert_block(db, block); - // Execute block 10 using an *internal* txn + // Execute block 11 using an *external* txn, then commit + db::RWTxnManaged external_txn0{db}; BlockNum start_block{10}, end_block{10}; - const auto result0{execute_blocks(nullptr, start_block, end_block)}; + const auto result0{execute_blocks(*external_txn0, start_block, end_block)}; + CHECK_NOTHROW(external_txn0.commit_and_stop()); CHECK(result0.execute_block_result == SILKWORM_OK); CHECK(result0.last_executed_block == end_block); CHECK(result0.mdbx_error_code == 0); @@ -279,11 +348,94 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks single block: OK", "[si insert_block(db, block); // Execute block 11 using an *external* txn, then commit - db::RWTxnManaged external_txn{db}; + db::RWTxnManaged external_txn1{db}; start_block = 11, end_block = 11; - const auto result1{execute_blocks(*external_txn, start_block, end_block)}; - CHECK_NOTHROW(external_txn.commit_and_stop()); + const auto result1{execute_blocks(*external_txn1, start_block, end_block)}; + CHECK_NOTHROW(external_txn1.commit_and_stop()); + CHECK(result1.execute_block_result == SILKWORM_OK); + CHECK(result1.last_executed_block == end_block); + CHECK(result1.mdbx_error_code == 0); + + ro_txn = db::ROTxnManaged{db}; + REQUIRE(db::read_account(ro_txn, to)); + CHECK(db::read_account(ro_txn, to)->balance == 2 * value); +} + +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_perpetual single block: OK", "[silkworm][capi]") { + // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII + SilkwormLibrary silkworm_lib{db.get_path()}; + + const int chain_id{1}; + const uint64_t batch_size{256 * kMebi}; + const bool write_change_sets{false}; // We CANNOT write changesets here, TestDatabaseContext db already has them + const bool write_receipts{false}; // We CANNOT write receipts here, TestDatabaseContext db already has them + const bool write_call_traces{false}; // For coherence but don't care + + auto execute_blocks = [&](auto start_block, auto end_block) { + return silkworm_lib.execute_blocks_perpetual(db, + chain_id, + start_block, + end_block, + batch_size, + write_change_sets, + write_receipts, + write_call_traces); + }; + + /* TestDatabaseContext db contains a test chain made up of 9 blocks */ + + // Prepare and insert block 10 (just 1 tx w/ value transfer) + evmc::address from{0x658bdf435d810c91414ec09147daa6db62406379_address}; // funded in genesis + evmc::address to{0x8b299e2b7d7f43c0ce3068263545309ff4ffb521_address}; // untouched address + intx::uint256 value{1 * kEther}; + + Block block{}; + block.header.number = 10; + block.header.gas_limit = 5'000'000; + block.header.gas_used = 21'000; + + static constexpr auto kEncoder = [](Bytes& dest, const Receipt& r) { rlp::encode(dest, r); }; + std::vector receipts{ + {TransactionType::kLegacy, true, block.header.gas_used, {}, {}}, + }; + block.header.receipts_root = trie::root_hash(receipts, kEncoder); + block.transactions.resize(1); + block.transactions[0].to = to; + block.transactions[0].gas_limit = block.header.gas_limit; + block.transactions[0].type = TransactionType::kLegacy; + block.transactions[0].max_priority_fee_per_gas = 0; + block.transactions[0].max_fee_per_gas = 20 * kGiga; + block.transactions[0].value = value; + block.transactions[0].r = 1; // dummy + block.transactions[0].s = 1; // dummy + block.transactions[0].set_sender(from); + + insert_block(db, block); + + // Execute block 10 using an *internal* txn + BlockNum start_block{10}, end_block{10}; + const auto result0{execute_blocks(start_block, end_block)}; + CHECK(result0.execute_block_result == SILKWORM_OK); + CHECK(result0.last_executed_block == end_block); + CHECK(result0.mdbx_error_code == 0); + + db::ROTxnManaged ro_txn{db}; + REQUIRE(db::read_account(ro_txn, to)); + CHECK(db::read_account(ro_txn, to)->balance == value); + ro_txn.abort(); + + // Prepare and insert block 11 (same as block 10) + block.transactions.erase(block.transactions.cbegin()); + block.transactions.pop_back(); + block.header.number = 11; + block.transactions[0].nonce++; + + insert_block(db, block); + + // Execute block 11 using an *internal* txn + start_block = 11, end_block = 11; + const auto result1{execute_blocks(start_block, end_block)}; CHECK(result1.execute_block_result == SILKWORM_OK); CHECK(result1.last_executed_block == end_block); CHECK(result1.mdbx_error_code == 0); @@ -293,7 +445,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks single block: OK", "[si CHECK(db::read_account(ro_txn, to)->balance == 2 * value); } -TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks multiple blocks: OK", "[silkworm][capi]") { +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_ephemeral multiple blocks: OK", "[silkworm][capi]") { // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII SilkwormLibrary silkworm_lib{db.get_path()}; @@ -304,8 +456,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks multiple blocks: OK", " const bool write_call_traces{false}; // For coherence but don't care auto execute_blocks = [&](auto tx, auto start_block, auto end_block) { - return silkworm_lib.execute_blocks(db, - tx, + return silkworm_lib.execute_blocks(tx, chain_id, start_block, end_block, @@ -353,9 +504,11 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks multiple blocks: OK", " block.transactions[0].nonce++; } - // Execute N blocks using an *internal* txn + // Execute N blocks using an *external* txn, then commit + db::RWTxnManaged external_txn0{db}; BlockNum start_block{10}, end_block{10 + kBlocks - 1}; - const auto result0{execute_blocks(nullptr, start_block, end_block)}; + const auto result0{execute_blocks(*external_txn0, start_block, end_block)}; + CHECK_NOTHROW(external_txn0.commit_and_stop()); CHECK(result0.execute_block_result == SILKWORM_OK); CHECK(result0.last_executed_block == end_block); CHECK(result0.mdbx_error_code == 0); @@ -375,11 +528,103 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks multiple blocks: OK", " } // Execute N blocks using an *external* txn, then commit - db::RWTxnManaged external_txn{db}; + db::RWTxnManaged external_txn1{db}; start_block = 10 + kBlocks, end_block = 10 + 2 * kBlocks - 1; - const auto result1{execute_blocks(*external_txn, start_block, end_block)}; - CHECK_NOTHROW(external_txn.commit_and_stop()); + const auto result1{execute_blocks(*external_txn1, start_block, end_block)}; + CHECK_NOTHROW(external_txn1.commit_and_stop()); + CHECK(result1.execute_block_result == SILKWORM_OK); + CHECK(result1.last_executed_block == end_block); + CHECK(result1.mdbx_error_code == 0); + + ro_txn = db::ROTxnManaged{db}; + REQUIRE(db::read_account(ro_txn, to)); + CHECK(db::read_account(ro_txn, to)->balance == 2 * kBlocks * value); +} + +TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_perpetual multiple blocks: OK", "[silkworm][capi]") { + // Use Silkworm as a library with silkworm_init/silkworm_fini automated by RAII + SilkwormLibrary silkworm_lib{db.get_path()}; + + const int chain_id{1}; + const uint64_t batch_size{256 * kMebi}; + const bool write_change_sets{false}; // We CANNOT write changesets here, TestDatabaseContext db already has them + const bool write_receipts{false}; // We CANNOT write receipts here, TestDatabaseContext db already has them + const bool write_call_traces{false}; // For coherence but don't care + + auto execute_blocks = [&](auto start_block, auto end_block) { + return silkworm_lib.execute_blocks_perpetual(db, + chain_id, + start_block, + end_block, + batch_size, + write_change_sets, + write_receipts, + write_call_traces); + }; + + /* TestDatabaseContext db contains a test chain made up of 9 blocks */ + + // Prepare block template (just 1 tx w/ value transfer) + evmc::address from{0x658bdf435d810c91414ec09147daa6db62406379_address}; // funded in genesis + evmc::address to{0x8b299e2b7d7f43c0ce3068263545309ff4ffb521_address}; // untouched address + intx::uint256 value{1}; + + Block block{}; + block.header.gas_limit = 5'000'000; + block.header.gas_used = 21'000; + + static constexpr auto kEncoder = [](Bytes& dest, const Receipt& r) { rlp::encode(dest, r); }; + std::vector receipts{ + {TransactionType::kLegacy, true, block.header.gas_used, {}, {}}, + }; + block.header.receipts_root = trie::root_hash(receipts, kEncoder); + block.transactions.resize(1); + block.transactions[0].to = to; + block.transactions[0].gas_limit = block.header.gas_limit; + block.transactions[0].type = TransactionType::kLegacy; + block.transactions[0].max_priority_fee_per_gas = 0; + block.transactions[0].max_fee_per_gas = 20 * kGiga; + block.transactions[0].value = value; + block.transactions[0].r = 1; // dummy + block.transactions[0].s = 1; // dummy + block.transactions[0].set_sender(from); + + constexpr size_t kBlocks{130}; + + // Insert N blocks + for (size_t i{10}; i < 10 + kBlocks; ++i) { + block.header.number = i; + insert_block(db, block); + block.transactions.erase(block.transactions.cbegin()); + block.transactions.pop_back(); + block.transactions[0].nonce++; + } + + // Execute N blocks using an *internal* txn + BlockNum start_block{10}, end_block{10 + kBlocks - 1}; + const auto result0{execute_blocks(start_block, end_block)}; + CHECK(result0.execute_block_result == SILKWORM_OK); + CHECK(result0.last_executed_block == end_block); + CHECK(result0.mdbx_error_code == 0); + + db::ROTxnManaged ro_txn{db}; + REQUIRE(db::read_account(ro_txn, to)); + CHECK(db::read_account(ro_txn, to)->balance == kBlocks * value); + ro_txn.abort(); + + // Insert N blocks again + for (size_t i{10 + kBlocks}; i < (10 + 2 * kBlocks); ++i) { + block.header.number = i; + insert_block(db, block); + block.transactions.erase(block.transactions.cbegin()); + block.transactions.pop_back(); + block.transactions[0].nonce++; + } + + // Execute N blocks using an *internal* txn, then commit + start_block = 10 + kBlocks, end_block = 10 + 2 * kBlocks - 1; + const auto result1{execute_blocks(start_block, end_block)}; CHECK(result1.execute_block_result == SILKWORM_OK); CHECK(result1.last_executed_block == end_block); CHECK(result1.mdbx_error_code == 0);