Changed default value of block-log-split

.gitlab-ci.yaml

@@ -338,6 +338,7 @@ mirrornet_replay_test:
   script:
     - echo "Generating artifacts file for block_log."
     - time ./hived
+      --block-log-split=-1
       -d "$MIRRORNET_SOURCE_5M_DATA_DIR"
     - echo "Compressing block log to $NUMBER_OF_BLOCKS blocks with $NUMBER_OF_PROCESSES processes"
     - time ./compress_block_log
@@ -357,6 +358,7 @@ mirrornet_replay_test:
       --jobs $NUMBER_OF_PROCESSES
     - echo "Starting hived replay"
     - ./hived
+      --block-log-split=-1
       -d "$MIRRORNET_BLOCKCHAIN_DATA_DIR"
       --chain-id $MIRRORNET_CHAIN_ID
       --skeleton-key "$MIRRORNET_SKELETON_KEY"
@@ -473,7 +475,7 @@ formatting_with_black_check:
   after_script:
     - |
       shopt -s globstar
-      tar --exclude='**/generated_during_*/**/block_log' --exclude='**/generated_during_*/**/block_log.artifacts' --exclude='**/generated_during_*/**/shared_memory.bin' --exclude='**/generated_during_*/**/*.sst' -cf - **/generated_during_* | 7z a -si generated_during.tar.7z
+      tar --exclude='**/generated_during_*/**/block_log*' --exclude='**/generated_during_*/**/shared_memory.bin' --exclude='**/generated_during_*/**/*.sst' -cf - **/generated_during_* | 7z a -si generated_during.tar.7z
       tar -cf - **/generated_by_package_fixtures | 7z a -si generated_by_package_fixtures.tar.7z
   artifacts:
     reports:
@@ -823,8 +825,7 @@ python_pattern_mainnet_tests:
     - "**/generated_during_*"
     - "**/generated_by_package_fixtures"
     exclude:
-    - "**/generated_during_*/**/block_log"
-    - "**/generated_during_*/**/block_log.artifacts"
+    - "**/generated_during_*/**/block_log*"
     - "**/generated_during_*/**/shared_memory.bin"
     - "**/generated_during_*/**/*.sst"
     when: always
@@ -896,8 +897,7 @@ message_format_mainnet_5m_tests:
     - "**/generated_during_*"
     - "**/generated_by_package_fixtures"
     exclude:
-    - "**/generated_during_*/**/block_log"
-    - "**/generated_during_*/**/block_log.artifacts"
+    - "**/generated_during_*/**/block_log*"
     - "**/generated_during_*/**/shared_memory.bin"
     - "**/generated_during_*/**/*.sst"
     when: always

libraries/chain/block_log_wrapper.cpp

@@ -258,12 +258,12 @@ std::shared_ptr<full_block_type> block_log_wrapper::get_block_by_number( uint32_
     return std::shared_ptr<full_block_type>();
 
   const block_log_ptr_t log = get_block_log_corresponding_to( block_num );
-  if( _block_log_split < MAX_FILES_OF_SPLIT_BLOCK_LOG )
+  if( _block_log_split <= MAX_FILES_OF_SPLIT_BLOCK_LOG )
   {
     FC_ASSERT( log,
       "Block ${num} has been pruned (oldest stored block is ${old}). "
       "Consider disabling pruning or increasing block-log-split value (currently ${part_count}).",
-      ("num", block_num)("old", get_tail_block_num())("part_count", _block_log_split) );
+      ("num", block_num)("old", get_actual_tail_block_num())("part_count", _block_log_split) );
   }
   else
   {
@@ -392,6 +392,7 @@ block_id_type block_log_wrapper::read_block_id_by_num( uint32_t block_num ) cons
 const block_log_wrapper::block_log_ptr_t block_log_wrapper::get_block_log_corresponding_to(
   uint32_t block_num ) const
 {
+  FC_ASSERT( get_head_log(), "Using unopened block log?" );
   uint32_t request_part_number = get_part_number_for_block( block_num, _max_blocks_in_log_file );
   if( request_part_number > _logs.size() )
     return block_log_ptr_t();
@@ -410,7 +411,12 @@ block_log_wrapper::full_block_range_t block_log_wrapper::read_block_range_by_num
     if( not current_log )
       return result;
 
-    auto part_result = current_log->read_block_range_by_num( starting_block_num, count );
+    FC_ASSERT( current_log->head(), "Empty or unopened log! ${io}", ("io", current_log->is_open() ) );
+
+    // block_log class is not prepared for over-the-head count parameter.
+    uint32_t actual_count =
+      std::min<uint32_t>( count, current_log->head()->get_block_num() - starting_block_num +1);
+    auto part_result = current_log->read_block_range_by_num( starting_block_num, actual_count );
     size_t part_result_count = part_result.size();
     if( part_result_count == 0 )
       return result;
@@ -584,27 +590,13 @@ void block_log_wrapper::common_open_and_init( bool read_only, bool allow_splitti
   if( _block_log_split == LEGACY_SINGLE_FILE_BLOCK_LOG )
   {
     auto single_part_log = std::make_shared<block_log>( _app );
-    internal_open_and_init( single_part_log, 
-                            _open_args.data_dir / block_log_file_name_info::_legacy_file_name,
-                            read_only );
+    auto log_file_path = _open_args.data_dir / block_log_file_name_info::_legacy_file_name;
+    ilog("Opening monolithic block log file ${log_file_path}", (log_file_path));
+    internal_open_and_init( single_part_log, log_file_path, read_only );
     _logs.push_back( single_part_log );
     return;
   }
 
-  uint32_t actual_tail_number_needed = 0;
-  if( _open_args.load_snapshot || _open_args.force_replay )
-  {
-    // When hard replay is in cards, ignore current state head (will be overridden anyway),
-    // and split as many parts as it's possible from the source log.
-    actual_tail_number_needed = 1;
-  }
-  else if ( _open_args.replay && not _open_args.force_replay && state_head_block )
-  {
-    // For regular replay require all parts beginning from state head block to be present & opened.
-    actual_tail_number_needed = 
-      get_part_number_for_block( state_head_block->get_block_num(), _max_blocks_in_log_file );
-  }
-
   // Any log file created on previous run?
   part_file_names_t part_file_names;
   look_for_part_files( part_file_names );
@@ -656,26 +648,42 @@ void block_log_wrapper::common_open_and_init( bool read_only, bool allow_splitti
 
   // Make sure all parts required by configuration are there.
   uint32_t head_part_number = part_file_names.crbegin()->part_number;
-  // Determine actual needed tail part number (if not set earlier).
-  if( actual_tail_number_needed == 0 )
+  // Determine actual needed tail part number.
+  uint32_t actual_tail_number_needed = head_part_number;
+  if( _open_args.load_snapshot || _open_args.force_replay ||
+      head_part_number <= (unsigned int)_block_log_split )
+  {
+    // When hard replay is in cards, ignore current state head (will be overridden anyway),
+    // and split as many parts as it's possible from the source log.
+    // Do the same when block log is effectively not pruned (yet).
+    actual_tail_number_needed = 1;
+  }
+  else
   {
-    // Is block log not pruned or effectively not pruned (yet)?
-    if( _block_log_split == MAX_FILES_OF_SPLIT_BLOCK_LOG ||
-        head_part_number <= (unsigned int)_block_log_split )
+    if ( _open_args.replay && not _open_args.force_replay && state_head_block )
     {
-      // Expected tail part is obviously 1 - we need each part.
-      actual_tail_number_needed = 1;
+      // For regular replay require all parts beginning from state head block to be present & opened.
+      actual_tail_number_needed = 
+        get_part_number_for_block( state_head_block->get_block_num(), _max_blocks_in_log_file );
     }
-    else // pruned log here
+
+    if( head_part_number > (uint32_t)_block_log_split )
     {
-      // not all parts are needed here, i.e. head_part_number > _block_log_split
-      actual_tail_number_needed = head_part_number - _block_log_split;
+      // Make sure that we require sufficient number of parts.
+      actual_tail_number_needed = 
+        std::min<uint32_t>( actual_tail_number_needed, head_part_number - _block_log_split);
     }
   }
   force_parts_exist( head_part_number, actual_tail_number_needed, part_file_names,
                       allow_splitting_monolithic_log, state_head_block );
 
   // Open all needed parts.
+  if( actual_tail_number_needed < head_part_number )
+    ilog( "Opening split block log files numbered ${from} to ${to}",
+          ("from", actual_tail_number_needed)("to", head_part_number) );
+  else
+    ilog( "Opening single split block log file numbered ${to}", ("to", head_part_number) );
+
   _logs.resize( head_part_number, block_log_ptr_t() );
   for( auto cit = part_file_names.cbegin(); cit != part_file_names.cend(); ++cit )
   {
@@ -766,16 +774,31 @@ void block_log_wrapper::internal_append( uint32_t block_num, append_t do_appendi
   }
 }
 
-uint32_t block_log_wrapper::get_tail_block_num() const
+uint32_t block_log_wrapper::get_actual_tail_block_num() const
 {
-  if( _block_log_split == LEGACY_SINGLE_FILE_BLOCK_LOG ||
-      _block_log_split == MAX_FILES_OF_SPLIT_BLOCK_LOG )
+  //FC_ASSERT( get_head_log(), "Using unopened block log?" );
+  size_t log_number = _logs.size();
+  FC_ASSERT( log_number > 0, "Uninitialized block_log_wrapper object?" );
+  FC_ASSERT( std::atomic_load( &(_logs[log_number -1]) ), "Head log not initialized?" );
+
+  uint32_t block_num = 0;
+  do
   {
-    return 1;
+    block_num = 1 + (log_number-1) * BLOCKS_IN_SPLIT_BLOCK_LOG_FILE;
+    --log_number;
+  } while ( log_number > 0 && std::atomic_load( &(_logs[log_number -1]) ) );
+
+  return block_num;
+
+  /*uint32_t block_num = 1;
+  block_log_ptr_t another_log;
+  for(; not another_log; block_num += BLOCKS_IN_SPLIT_BLOCK_LOG_FILE )
+  {
+    another_log = get_block_log_corresponding_to( block_num );
   }
+  while( not another_log ); // Will finally happen, see initial assertion.
 
-  int oldest_available_part = std::max<int>( _logs.size() - _block_log_split, 1 );
-  return (oldest_available_part -1) * BLOCKS_IN_SPLIT_BLOCK_LOG_FILE + 1;
+  return block_num;*/
 }
 
 } } //hive::chain

libraries/chain/include/hive/chain/block_log_wrapper.hpp

@@ -174,7 +174,7 @@ namespace hive { namespace chain {
     }
 
     /// Returns the number of oldest stored block.
-    uint32_t get_tail_block_num() const;
+    uint32_t get_actual_tail_block_num() const;
 
     void dispose_garbage( bool closing_time );
 

libraries/plugins/chain/chain_plugin.cpp

@@ -1398,7 +1398,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
 #endif
       ("rc-stats-report-type", bpo::value<string>()->default_value( "REGULAR" ), "Level of detail of daily RC stat reports: NONE, MINIMAL, REGULAR, FULL. Default REGULAR." )
       ("rc-stats-report-output", bpo::value<string>()->default_value( "ILOG" ), "Where to put daily RC stat reports: DLOG, ILOG, NOTIFY, LOG_NOTIFY. Default ILOG." )
-      ("block-log-split", bpo::value<int>()->default_value( -1 ), "Whether the block log should be single file (-1), not used at all & keeping only head block in memory (0), or split into files each containing 1M blocks & keeping N full million latest blocks (N). Default -1." )
+      ("block-log-split", bpo::value<int>()->default_value( 9999 ), "Whether the block log should be single file (-1), not used at all & keeping only head block in memory (0), or split into files each containing 1M blocks & keeping N full million latest blocks (N). Default 9999." )
       ;
   cli.add_options()
       ("replay-blockchain", bpo::bool_switch()->default_value(false), "clear chain database and replay all blocks" )
@@ -1426,6 +1426,12 @@ void chain_plugin::plugin_initialize(const variables_map& options)
   my.reset( new detail::chain_plugin_impl( get_app() ) );
 
   my->block_log_split = options.at( "block-log-split" ).as< int >();
+  std::string block_storage_description( "single block in memory" );
+  if( my->block_log_split < 0 ) 
+    block_storage_description = "legacy monolithic file";
+  else if( my->block_log_split > 0 )
+    block_storage_description = "split into multiple files";
+  ilog("Block storage is configured to be ${bs}.", ("bs", block_storage_description));
   my->block_storage = block_storage_i::create_storage( my->block_log_split, get_app(), my->thread_pool );
   my->default_block_writer = 
     std::make_unique< sync_block_writer >( *( my->block_storage.get() ), my->db, get_app() );

tests/integration/artifacts_generation/test_artifacts_generation.sh

@@ -5,7 +5,10 @@ set -xeuo pipefail
 HIVED_BINARY_PATH=${1}
 BLOCK_LOG_UTIL_BINARY_PATH=${2}
 
+# Note that 100k blocks fit within a single split block log file
+# thus block_log.100k file can be used as both block_log & block_log_part.0001
 SOURCE_100K_BLOCK_LOG_PATTERN="blockchain/block_log.100k"
+# Same story with artifacts file, block_log.artifacts can be used as block_log_part.0001.artifacts here
 SOURCE_ARTIFACTS_V_1_0_PATTERN="blockchain/block_log.artifacts.v_1_0"
 SOURCE_ARTIFACTS_V_1_1_INTERRUPTED_PATTERN="blockchain/block_log.artifacts.v_1_1.interrupted_at_19313"
 
@@ -15,19 +18,26 @@ test_generate_artifacts_from_scratch_by_hived() {
   local TEST_BLOCKCHAIN_DIR="$TEST_DATA_DIR/blockchain"
   mkdir -p "$TEST_BLOCKCHAIN_DIR"
   cp "$SOURCE_100K_BLOCK_LOG_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log"
+  # Note that both block_log.artifacts & block_log_part.0001.artifacts are created below
+  # due to a combination of hived's default split setting & auto-split feature
   "$HIVED_BINARY_PATH" -d "$TEST_DATA_DIR" --replay --exit-before-sync
   "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log" --do-full-artifacts-verification-match-check --header-only
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001" --do-full-artifacts-verification-match-check --header-only
 }
 
 test_generate_artifacts_override_old_file_by_hived() {
   echo "Test 2 - override old artifacts file and create a new one by hived."
   local TEST_DATA_DIR="override_old_file_and_generate_from_scratch/by_hived"
   local TEST_BLOCKCHAIN_DIR="$TEST_DATA_DIR/blockchain"
   mkdir -p "$TEST_BLOCKCHAIN_DIR"
+  # Note that both block_log.artifacts & block_log_part.0001.artifacts are overridden below
+  # due to a combination of hived's default split setting & auto-split feature
   cp "$SOURCE_100K_BLOCK_LOG_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log"
   cp "$SOURCE_ARTIFACTS_V_1_0_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log.artifacts"
+  cp "$SOURCE_ARTIFACTS_V_1_0_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log_part.0001.artifacts"
   "$HIVED_BINARY_PATH" -d "$TEST_DATA_DIR" --replay --exit-before-sync
   "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log" --do-full-artifacts-verification-match-check --header-only
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001" --do-full-artifacts-verification-match-check --header-only
 }
 
 test_resume_artifacts_generating_process_by_hived() {
@@ -48,6 +58,9 @@ test_generate_artifacts_from_scratch_by_block_log_util() {
   cp "$SOURCE_100K_BLOCK_LOG_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log"
   "$BLOCK_LOG_UTIL_BINARY_PATH" --generate-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log"
   "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log" --do-full-artifacts-verification-match-check --header-only
+  cp "$SOURCE_100K_BLOCK_LOG_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log_part.0001"
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --generate-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001"
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001" --do-full-artifacts-verification-match-check --header-only
 }
 
 test_generate_artifacts_override_old_file_by_block_log_util() {
@@ -58,6 +71,10 @@ test_generate_artifacts_override_old_file_by_block_log_util() {
   cp "$SOURCE_ARTIFACTS_V_1_0_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log.artifacts"
   "$BLOCK_LOG_UTIL_BINARY_PATH" --generate-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log"
   "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log" --do-full-artifacts-verification-match-check --header-only
+  cp "$SOURCE_100K_BLOCK_LOG_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log_part.0001"
+  cp "$SOURCE_ARTIFACTS_V_1_0_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log_part.0001.artifacts"
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --generate-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001"
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001" --do-full-artifacts-verification-match-check --header-only
 }
 
 test_resume_artifacts_generating_process_by_block_log_util() {
@@ -68,6 +85,10 @@ test_resume_artifacts_generating_process_by_block_log_util() {
   cp "$SOURCE_ARTIFACTS_V_1_1_INTERRUPTED_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log.artifacts"
   "$BLOCK_LOG_UTIL_BINARY_PATH" --generate-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log"
   "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log" --do-full-artifacts-verification-match-check --header-only
+  cp "$SOURCE_100K_BLOCK_LOG_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log_part.0001"
+  cp "$SOURCE_ARTIFACTS_V_1_1_INTERRUPTED_PATTERN" "$TEST_BLOCKCHAIN_DIR/block_log_part.0001.artifacts"
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --generate-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001"
+  "$BLOCK_LOG_UTIL_BINARY_PATH" --get-block-artifacts --block-log "$TEST_BLOCKCHAIN_DIR/block_log_part.0001" --do-full-artifacts-verification-match-check --header-only
 }
 
 test_generate_artifacts_from_scratch_by_hived

tests/python/api_tests/python_patterns_tests/cli_wallet_tests/output_formatter_tests/conftest.py

@@ -12,10 +12,12 @@ def node(request):
     if request.node.get_closest_marker("replayed_node") is None:
         node = tt.InitNode()
         node.config.plugin.append("market_history_api")
+        node.config.block_log_split = -1
         node.run()
         return node
     api_node = tt.FullApiNode()
-    api_node.run(replay_from=Path(__file__).parent.joinpath("block_log/block_log"), wait_for_live=False)
+    api_node.config.block_log_split = -1
+    api_node.run(replay_from=Path(__file__).parent.joinpath("block_log"), wait_for_live=False)
     return api_node
 
 

tests/python/conftest.py

@@ -53,6 +53,7 @@ def __create_init_node() -> tt.InitNode:
         init_node.config.plugin.append(
             "condenser_api"
         )  # FIXME eliminate condenser_api usage from other tests than this API specific
+        init_node.config.block_log_split = -1
         init_node.run()
         return init_node
 

tests/python/functional/beem_tests/test_assets.py

@@ -16,12 +16,13 @@
 @pytest.fixture()
 def node(chain_id, skeleton_key):
     block_log_directory = Path(__file__).parent / "block_log_mirrornet_1k"
-    block_log = tt.BlockLog(block_log_directory / "block_log")
+    block_log = tt.BlockLog(block_log_directory, "auto")
 
     init_node = tt.InitNode()
     init_node.config.private_key = skeleton_key
     init_node.config.plugin.append("account_history_api")
     init_node.config.plugin.append("condenser_api")
+    init_node.config.block_log_split = -1
 
     init_node.run(
         time_control=tt.StartTimeControl(start_time="head_block_time"),

tests/python/functional/colony/conftest.py

@@ -7,10 +7,10 @@
 
 
 @pytest.fixture()
-def alternate_chain_spec(block_log) -> tt.AlternateChainSpecs:
-    return tt.AlternateChainSpecs.parse_file(block_log.path.parent / tt.AlternateChainSpecs.FILENAME)
+def alternate_chain_spec(block_log_single_sign) -> tt.AlternateChainSpecs:
+    return tt.AlternateChainSpecs.parse_file(block_log_single_sign.path / tt.AlternateChainSpecs.FILENAME)
 
 
 @pytest.fixture()
-def block_log() -> tt.BlockLog:
-    return tt.BlockLog(path=UNIVERSAL_BLOCK_LOGS_PATH / "block_log_single_sign" / "block_log")
+def block_log_single_sign() -> tt.BlockLog:
+    return tt.BlockLog(path=UNIVERSAL_BLOCK_LOGS_PATH / "block_log_single_sign", mode="auto")