From 77309a0e1040f01b39b591a0fde5f086b007c20b Mon Sep 17 00:00:00 2001 From: RubenD Date: Fri, 31 May 2024 14:50:21 -0400 Subject: [PATCH 1/2] adding compaction from lib --- src/lib.rs | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 2b3d4b8..ed82e85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -167,9 +167,52 @@ impl<'a> From<(Bound>, Bound>)> for CowStringRange<'a> } } +/// Perform a compaction of the DB. +/// +/// Written for use with downstream crates needing periodic +/// compactions. +pub fn compact(dir: &std::path::Path, major: bool) -> std::io::Result<()> { + use fs2::FileExt; + let lock = std::fs::File::create(dir.join(".compact"))?; + lock.try_lock_exclusive()?; + + let db = if major { + DatabaseReader::new(dir)? + } else { + DatabaseReader::without_main_db(dir)? + }; + + let mut no_transaction_paths = false; + { + let ps = db.transaction_paths(); + if ps.len() == 1 && ps[0].file_name().expect("filename") == "main" { + no_transaction_paths = true; + } + } + + if db.num_txes() <= 1 || no_transaction_paths { + println!("Compaction has nothing to do."); + } else { + println!("Processing {} .txes", db.num_txes()); + let db = std::sync::Arc::new(db); + let mut compacted = CreateTx::new(dir)?; + + // create the new transaction after opening the database reader + let reader = db.get_range(..); + let mut n = 0u64; + for record in reader { + compacted.add_record_raw(record.key(), record.format(), record.raw()).expect("Error adding record"); + n += 1; + } + _purge_compacted_files(compacted, dir, &db, major)?; + println!("Compacted {} records.", {n}); + } + + Ok(()) +} + // not part of public api #[doc(hidden)] -#[cfg(feature = "bin")] pub fn _purge_compacted_files( compacted: CreateTx, dir: &std::path::Path, From 25add5905ea219d9a45554d155478c257fb4e653 Mon Sep 17 00:00:00 2001 From: RubenD Date: Fri, 31 May 2024 15:10:25 -0400 Subject: [PATCH 2/2] returns # records compacted --- src/lib.rs | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ed82e85..a99a162 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,10 +168,10 @@ impl<'a> From<(Bound>, Bound>)> for CowStringRange<'a> } /// Perform a compaction of the DB. -/// -/// Written for use with downstream crates needing periodic -/// compactions. -pub fn compact(dir: &std::path::Path, major: bool) -> std::io::Result<()> { +/// +/// For use with downstream crates needing periodic +/// compactions. Returns the number of records compacted. +pub fn compact(dir: &std::path::Path, major: bool) -> std::io::Result { use fs2::FileExt; let lock = std::fs::File::create(dir.join(".compact"))?; lock.try_lock_exclusive()?; @@ -182,33 +182,28 @@ pub fn compact(dir: &std::path::Path, major: bool) -> std::io::Result<()> { DatabaseReader::without_main_db(dir)? }; - let mut no_transaction_paths = false; { let ps = db.transaction_paths(); - if ps.len() == 1 && ps[0].file_name().expect("filename") == "main" { - no_transaction_paths = true; + if db.num_txes() <= 1 || (ps.len() == 1 && ps[0].file_name().expect("filename") == "main") { + return Ok(0); } } - - if db.num_txes() <= 1 || no_transaction_paths { - println!("Compaction has nothing to do."); - } else { - println!("Processing {} .txes", db.num_txes()); - let db = std::sync::Arc::new(db); - let mut compacted = CreateTx::new(dir)?; - - // create the new transaction after opening the database reader - let reader = db.get_range(..); - let mut n = 0u64; - for record in reader { - compacted.add_record_raw(record.key(), record.format(), record.raw()).expect("Error adding record"); - n += 1; - } - _purge_compacted_files(compacted, dir, &db, major)?; - println!("Compacted {} records.", {n}); + // println!("Processing {} .txes", db.num_txes()); + let db = std::sync::Arc::new(db); + let mut compacted = CreateTx::new(dir)?; + + // create the new transaction after opening the database reader + let reader = db.get_range(..); + let mut n = 0u64; + for record in reader { + compacted + .add_record_raw(record.key(), record.format(), record.raw()) + .expect("Error adding record"); + n += 1; } + _purge_compacted_files(compacted, dir, &db, major)?; - Ok(()) + Ok(n) } // not part of public api