feat: Add compact database tool (#9341)

Add a database tool for running SST file compactions. This is useful when changing block size parameters in order to have an optimized layout of data in SST files.

core/store/src/db/rocksdb.rs

@@ -377,6 +377,7 @@ impl Database for RocksDB {
     fn compact(&self) -> io::Result<()> {
         let none = Option::<&[u8]>::None;
         for col in DBCol::iter() {
+            tracing::info!(target: "db", column = %col, "Compacted column");
             self.db.compact_range_cf(self.cf_handle(col)?, none, none);
         }
         Ok(())

tools/database/README.md

@@ -62,6 +62,18 @@ Then you can call
 `neard database change-db-kind --new-kind Cold change-hot`.
 Notice that even though in your mind this db is cold, in your config this db hot, so you have to pass `change-hot`.
 
+## Compact database
+
+Run compaction on the SST files. Running this command might increase database read performance.
+This is good use case when changing `block_size` and wishing to perform test on how the RocksDB performance has
+changed.
+
+Example usage:
+```bash
+cargo run --bin neard -- database compact-database
+```
+
+
 ## Make a DB Snapshot
 
 Makes a copy of a DB (hot store only) at a specified location. If the
@@ -70,7 +82,7 @@ take no additional disk space due to hardlinking all the files.
 
 Example usage:
 ```bash
-cargo run --bin neard -- --home /home/ubuntu/.near database make_snapshot --destination /home/ubuntu/.near/data/snapshot
+cargo run --bin neard -- --home /home/ubuntu/.near database make-snapshot --destination /home/ubuntu/.near/data/snapshot
 ```
 
 In this example all `.sst` files from `/home/ubuntu/.near/data` will be also

tools/database/src/analyse_data_size_distribution.rs

@@ -197,7 +197,7 @@ fn get_column_families(input_col: &Option<String>) -> Vec<DBCol> {
 
 impl AnalyseDataSizeDistributionCommand {
     pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> {
-        let db = open_rocksdb(home)?;
+        let db = open_rocksdb(home, near_store::Mode::ReadOnly)?;
         let column_families = get_column_families(&self.column);
         let results = read_all_pairs(&db, &column_families);
         results.print_results(self.top_k);

tools/database/src/commands.rs

@@ -1,5 +1,6 @@
 use crate::adjust_database::ChangeDbKindCommand;
 use crate::analyse_data_size_distribution::AnalyseDataSizeDistributionCommand;
+use crate::compact::RunCompactionCommand;
 use crate::make_snapshot::MakeSnapshotCommand;
 use crate::run_migrations::RunMigrationsCommand;
 use crate::state_perf::StatePerfCommand;
@@ -21,6 +22,9 @@ enum SubCommand {
     /// Change DbKind of hot or cold db.
     ChangeDbKind(ChangeDbKindCommand),
 
+    /// Run SST file compaction on database
+    CompactDatabase(RunCompactionCommand),
+
     /// Make snapshot of the database
     MakeSnapshot(MakeSnapshotCommand),
 
@@ -37,6 +41,7 @@ impl DatabaseCommand {
         match &self.subcmd {
             SubCommand::AnalyseDataSizeDistribution(cmd) => cmd.run(home),
             SubCommand::ChangeDbKind(cmd) => cmd.run(home),
+            SubCommand::CompactDatabase(cmd) => cmd.run(home),
             SubCommand::MakeSnapshot(cmd) => {
                 let near_config = nearcore::config::load_config(
                     &home,

tools/database/src/compact.rs

@@ -0,0 +1,16 @@
+use crate::utils::open_rocksdb;
+use clap::Parser;
+use near_store::db::Database;
+use std::path::PathBuf;
+
+#[derive(Parser)]
+pub(crate) struct RunCompactionCommand {}
+
+impl RunCompactionCommand {
+    pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> {
+        let db = open_rocksdb(home, near_store::Mode::ReadWrite)?;
+        db.compact()?;
+        eprintln!("Compaction is finished!");
+        Ok(())
+    }
+}

tools/database/src/lib.rs

@@ -1,6 +1,7 @@
 mod adjust_database;
 mod analyse_data_size_distribution;
 pub mod commands;
+mod compact;
 mod make_snapshot;
 mod run_migrations;
 mod state_perf;

tools/database/src/state_perf.rs

@@ -32,7 +32,7 @@ pub(crate) struct StatePerfCommand {
 
 impl StatePerfCommand {
     pub(crate) fn run(&self, home: &Path) -> anyhow::Result<()> {
-        let rocksdb = Arc::new(open_rocksdb(home)?);
+        let rocksdb = Arc::new(open_rocksdb(home, near_store::Mode::ReadOnly)?);
         let store = near_store::NodeStorage::new(rocksdb).get_hot_store();
         eprintln!("Start State perf test");
         let mut perf_context = PerfContext::new();

tools/database/src/utils.rs

@@ -1,16 +1,15 @@
 use std::path::Path;
 
-pub(crate) fn open_rocksdb(home: &Path) -> anyhow::Result<near_store::db::RocksDB> {
+pub(crate) fn open_rocksdb(
+    home: &Path,
+    mode: near_store::Mode,
+) -> anyhow::Result<near_store::db::RocksDB> {
     let config = nearcore::config::Config::from_file_skip_validation(
         &home.join(nearcore::config::CONFIG_FILENAME),
     )?;
     let store_config = &config.store;
     let db_path = store_config.path.as_ref().cloned().unwrap_or_else(|| home.join("data"));
-    let rocksdb = near_store::db::RocksDB::open(
-        &db_path,
-        store_config,
-        near_store::Mode::ReadOnly,
-        near_store::Temperature::Hot,
-    )?;
+    let rocksdb =
+        near_store::db::RocksDB::open(&db_path, store_config, mode, near_store::Temperature::Hot)?;
     Ok(rocksdb)
 }