From a2bfdee93b77f2e70c8bfbed1f1c1f98a82179e9 Mon Sep 17 00:00:00 2001 From: Matus Kysel Date: Fri, 12 May 2023 10:12:53 +0200 Subject: [PATCH] feat: clean LOG.old* files on db start --- .../storage/include/storage/storage.hpp | 4 +++ libraries/core_libs/storage/src/storage.cpp | 30 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/libraries/core_libs/storage/include/storage/storage.hpp b/libraries/core_libs/storage/include/storage/storage.hpp index 8d6f00182d..60986942e4 100644 --- a/libraries/core_libs/storage/include/storage/storage.hpp +++ b/libraries/core_libs/storage/include/storage/storage.hpp @@ -178,6 +178,10 @@ class DbStorage : public std::enable_shared_from_this { void enableSnapshots(); void updateDbVersions(); + // For removal of LOG.old.* files in the database + void removeOldLogFiles() const; + void removeFilesWithPattern(const std::string& directory, const std::regex& pattern) const; + uint32_t getMajorVersion() const; std::unique_ptr getColumnIterator(const Column& c); diff --git a/libraries/core_libs/storage/src/storage.cpp b/libraries/core_libs/storage/src/storage.cpp index aeb076e2b8..160529236a 100644 --- a/libraries/core_libs/storage/src/storage.cpp +++ b/libraries/core_libs/storage/src/storage.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "config/version.hpp" #include "dag/sortition_params_manager.hpp" @@ -42,6 +43,8 @@ DbStorage::DbStorage(fs::path const& path, uint32_t db_snapshot_each_n_pbft_bloc } fs::create_directories(db_path_); + removeOldLogFiles(); + rocksdb::Options options; options.create_missing_column_families = true; options.create_if_missing = true; @@ -86,6 +89,26 @@ DbStorage::DbStorage(fs::path const& path, uint32_t db_snapshot_each_n_pbft_bloc } } +void DbStorage::removeOldLogFiles() const { + const std::regex filePattern("LOG\\.old\\.\\d+"); + removeFilesWithPattern(db_path_, filePattern); + removeFilesWithPattern(state_db_path_, filePattern); +} + +void DbStorage::removeFilesWithPattern(const std::string& directory, const std::regex& pattern) const { + try { + for (const auto& entry : std::filesystem::directory_iterator(directory)) { + const std::string& filename = entry.path().filename().string(); + if (std::regex_match(filename, pattern)) { + std::filesystem::remove(entry.path()); + LOG(log_dg_) << "Removed file: " << filename << std::endl; + } + } + } catch (const std::filesystem::filesystem_error& e) { + LOG(log_dg_) << "Error accessing directory: " << e.what() << std::endl; + } +} + void DbStorage::updateDbVersions() { saveStatusField(StatusDbField::DbMajorVersion, TARAXA_DB_MAJOR_VERSION); saveStatusField(StatusDbField::DbMinorVersion, TARAXA_DB_MINOR_VERSION); @@ -454,7 +477,8 @@ void DbStorage::clearPeriodDataHistory(PbftPeriod end_period) { auto start_slice = toSlice(start_period); auto end_slice = toSlice(end_period); for (auto period = start_period; period < end_period; period++) { - // Find transactions included in the old blocks and delete data related to these transactions to free disk space + // Find transactions included in the old blocks and delete data related to these transactions to free disk + // space auto trx_hashes_raw = lookup(period, DB::Columns::final_chain_transaction_hashes_by_blk_number); auto hashes_count = trx_hashes_raw.size() / trx_hash_t::size; for (uint32_t i = 0; i < hashes_count; i++) { @@ -471,8 +495,8 @@ void DbStorage::clearPeriodDataHistory(PbftPeriod end_period) { commitWriteBatch(write_batch); db_->DeleteRange(write_options_, handle(Columns::period_data), start_slice, end_slice); - // Deletion alone does not guarantee that the disk space is freed, these CompactRange methods actually compact the - // data in the database and free disk space + // Deletion alone does not guarantee that the disk space is freed, these CompactRange methods actually compact + // the data in the database and free disk space db_->CompactRange({}, handle(Columns::period_data), &start_slice, &end_slice); db_->CompactRange({}, handle(Columns::final_chain_receipt_by_trx_hash), nullptr, nullptr); db_->CompactRange({}, handle(Columns::final_chain_transaction_hashes_by_blk_number), nullptr, nullptr);