fix(bulk_load): rocksDB parameter allow_ingest_behind was lost after …

…replica migration (#1651)

#1650

While reopening a replica with empty envs(currently envs would be empty ONLY when reopening), put the last allow_ingest_behind value to the current option.

src/server/pegasus_server_impl.cpp

@@ -1667,8 +1667,8 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv)
             // We don't use `loaded_data_cf_opts` directly because pointer-typed options will
             // only be initialized with default values when calling 'LoadLatestOptions', see
             // 'rocksdb/utilities/options_util.h'.
-            reset_rocksdb_options(loaded_data_cf_opts, &_table_data_cf_opts);
-            _db_opts.allow_ingest_behind = parse_allow_ingest_behind(envs);
+            reset_rocksdb_options(
+                loaded_data_cf_opts, loaded_db_opt, envs, &_table_data_cf_opts, &_db_opts);
         }
     } else {
         // When create new DB, we have to create a new column family to store meta data (meta column
@@ -3134,8 +3134,11 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario)
     }
 }
 
-void pegasus_server_impl::reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts,
-                                                rocksdb::ColumnFamilyOptions *target_opts)
+void pegasus_server_impl::reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_cf_opts,
+                                                const rocksdb::DBOptions &base_db_opt,
+                                                const std::map<std::string, std::string> &envs,
+                                                rocksdb::ColumnFamilyOptions *target_cf_opts,
+                                                rocksdb::DBOptions *target_db_opt)
 {
     LOG_INFO_PREFIX("Reset rocksdb envs options");
     // Reset rocksdb option includes two aspects:
@@ -3144,11 +3147,13 @@ void pegasus_server_impl::reset_rocksdb_options(const rocksdb::ColumnFamilyOptio
     // ROCKSDB_STATIC_OPTIONS
 
     // aspect 1:
-    reset_usage_scenario_options(base_opts, target_opts);
+    reset_usage_scenario_options(base_cf_opts, target_cf_opts);
 
     // aspect 2:
-    target_opts->num_levels = base_opts.num_levels;
-    target_opts->write_buffer_size = base_opts.write_buffer_size;
+    target_cf_opts->num_levels = base_cf_opts.num_levels;
+    target_cf_opts->write_buffer_size = base_cf_opts.write_buffer_size;
+
+    reset_allow_ingest_behind_option(base_db_opt, envs, target_db_opt);
 }
 
 void pegasus_server_impl::reset_usage_scenario_options(
@@ -3169,6 +3174,19 @@ void pegasus_server_impl::reset_usage_scenario_options(
     target_opts->max_write_buffer_number = base_opts.max_write_buffer_number;
 }
 
+void pegasus_server_impl::reset_allow_ingest_behind_option(
+    const rocksdb::DBOptions &base_db_opt,
+    const std::map<std::string, std::string> &envs,
+    rocksdb::DBOptions *target_db_opt)
+{
+    if (envs.empty()) {
+        // for reopen db during load balance learning
+        target_db_opt->allow_ingest_behind = base_db_opt.allow_ingest_behind;
+    } else {
+        target_db_opt->allow_ingest_behind = parse_allow_ingest_behind(envs);
+    }
+}
+
 void pegasus_server_impl::recalculate_data_cf_options(
     const rocksdb::ColumnFamilyOptions &cur_data_cf_opts)
 {

src/server/pegasus_server_impl.h

@@ -366,8 +366,15 @@ class pegasus_server_impl : public pegasus_read_service
     void reset_usage_scenario_options(const rocksdb::ColumnFamilyOptions &base_opts,
                                       rocksdb::ColumnFamilyOptions *target_opts);
 
-    void reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts,
-                               rocksdb::ColumnFamilyOptions *target_opts);
+    void reset_allow_ingest_behind_option(const rocksdb::DBOptions &base_db_opt,
+                                          const std::map<std::string, std::string> &envs,
+                                          rocksdb::DBOptions *target_db_opt);
+
+    void reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_cf_opts,
+                               const rocksdb::DBOptions &base_db_opt,
+                               const std::map<std::string, std::string> &envs,
+                               rocksdb::ColumnFamilyOptions *target_cf_opts,
+                               rocksdb::DBOptions *target_db_opt);
 
     // return true if successfully set
     bool set_options(const std::unordered_map<std::string, std::string> &new_options);