From 02448c7cc260481d264bfd56fddc27e1ce71353c Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Tue, 3 Nov 2020 19:02:58 +0800 Subject: [PATCH] Refine some class/function names --- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 6 +-- dbms/src/Storages/DeltaMerge/Segment.cpp | 4 +- .../Storages/DeltaMerge/StableValueSpace.cpp | 2 +- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 6 +-- dbms/src/Storages/Page/PageStorage.cpp | 35 ++++++++++----- dbms/src/Storages/Page/PageStorage.h | 8 ++-- dbms/src/Storages/Page/gc/LegacyCompactor.cpp | 2 +- .../Page/tests/gtest_data_compactor.cpp | 2 +- .../Page/tests/gtest_legacy_compactor.cpp | 4 +- .../Page/tests/gtest_page_storage.cpp | 8 ++-- .../tests/gtest_page_storage_multi_paths.cpp | 6 +-- .../gtest_page_storage_multi_writers.cpp | 2 +- dbms/src/Storages/PathPool.cpp | 38 ++++++++-------- dbms/src/Storages/PathPool.h | 45 ++++++++++--------- dbms/src/Storages/tests/gtest_path_pool.cpp | 12 ++--- 15 files changed, 98 insertions(+), 82 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 72716ee6724..75eaf00c4e8 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -197,7 +197,7 @@ void DeltaMergeStore::setUpBackgroundTask(const DMContextPtr & dm_context) { auto dmfile_scanner = [=]() { PageStorage::PathAndIdsVec path_and_ids_vec; - auto delegate = path_pool.getStableDelegate(); + auto delegate = path_pool.getStableDiskDelegator(); for (auto & root_path : delegate.listPaths()) { auto & path_and_ids = path_and_ids_vec.emplace_back(); @@ -209,7 +209,7 @@ void DeltaMergeStore::setUpBackgroundTask(const DMContextPtr & dm_context) return path_and_ids_vec; }; auto dmfile_remover = [&](const PageStorage::PathAndIdsVec & path_and_ids_vec, const std::set & valid_ids) { - auto delegate = path_pool.getStableDelegate(); + auto delegate = path_pool.getStableDiskDelegator(); for (auto & [path, ids] : path_and_ids_vec) { for (auto id : ids) @@ -1496,7 +1496,7 @@ void DeltaMergeStore::restoreStableFiles() { LOG_DEBUG(log, "Loading dt files"); - auto path_delegate = path_pool.getStableDelegate(); + auto path_delegate = path_pool.getStableDiskDelegator(); for (const auto & root_path : path_delegate.listPaths()) { for (auto & file_id : DMFile::listAllInPath(global_context.getFileProvider(), root_path, false)) diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index fb458f509b5..9226917603f 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -111,7 +111,7 @@ DMFilePtr writeIntoNewDMFile(DMContext & dm_context, // StableValueSpacePtr createNewStable(DMContext & context, const BlockInputStreamPtr & input_stream, PageId stable_id, WriteBatches & wbs) { - auto delegate = context.path_pool.getStableDelegate(); + auto delegate = context.path_pool.getStableDiskDelegator(); auto store_path = delegate.choosePath(); PageId dmfile_id = context.storage_pool.newDataPageId(); @@ -716,7 +716,7 @@ Segment::prepareSplitLogical(DMContext & dm_context, const SegmentSnapshotPtr & DMFiles my_stable_files; DMFiles other_stable_files; - auto delegate = dm_context.path_pool.getStableDelegate(); + auto delegate = dm_context.path_pool.getStableDiskDelegator(); for (auto & dmfile : segment_snap->stable->getDMFiles()) { auto ori_ref_id = dmfile->refId(); diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp index c39e47b696d..c7197195b40 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp @@ -78,7 +78,7 @@ StableValueSpacePtr StableValueSpace::restore(DMContext & context, PageId id) readIntBinary(ref_id, buf); auto file_id = context.storage_pool.data().getNormalPageId(ref_id); - auto file_parent_path = context.path_pool.getStableDelegate().getDTFilePath(file_id); + auto file_parent_path = context.path_pool.getStableDiskDelegator().getDTFilePath(file_id); auto dmfile = DMFile::restore(context.db_context.getFileProvider(), file_id, ref_id, file_parent_path); stable->files.push_back(dmfile); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 3286c456348..efc7cba7512 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -35,19 +35,19 @@ PageStorage::Config extractConfig(const Settings & settings, UInt64 subtype) StoragePool::StoragePool(const String & name, StoragePathPool & path_pool, const Context & global_ctx, const Settings & settings) : // The iops and bandwidth in log_storage are relatively high, use multi-disks if possible log_storage(name + ".log", - path_pool.getMultiDiskDelegate("log"), + path_pool.getPSDiskDelegatorMulti("log"), extractConfig(settings, STORAGE_LOG), global_ctx.getFileProvider(), global_ctx.getTiFlashMetrics()), // The iops in data_storage is low, only use the first disk for storing data data_storage(name + ".data", - path_pool.getNormalDiskDelegate("data"), + path_pool.getPSDiskDelegatorSingle("data"), extractConfig(settings, STORAGE_DATA), global_ctx.getFileProvider(), global_ctx.getTiFlashMetrics()), // The iops in meta_storage is relatively high, use multi-disks if possible meta_storage(name + ".meta", - path_pool.getMultiDiskDelegate("meta"), + path_pool.getPSDiskDelegatorMulti("meta"), extractConfig(settings, STORAGE_META), global_ctx.getFileProvider(), global_ctx.getTiFlashMetrics()), diff --git a/dbms/src/Storages/Page/PageStorage.cpp b/dbms/src/Storages/Page/PageStorage.cpp index 0013cfb9184..ace4844d837 100644 --- a/dbms/src/Storages/Page/PageStorage.cpp +++ b/dbms/src/Storages/Page/PageStorage.cpp @@ -167,7 +167,7 @@ static inline bool isPageFileSizeFitsWritable(const PageFile & pf, const PageSto void PageStorage::restore() { LOG_INFO(log, - storage_name << " begin to restore data from disk. [path=" << delegator->normalPath() + storage_name << " begin to restore data from disk. [path=" << delegator->defaultPath() << "] [num_writers=" << write_files.size() << "]"); /// page_files are in ascending ordered by (file_id, level). @@ -293,7 +293,7 @@ void PageStorage::restore() std::vector store_paths = delegator->listPaths(); for (size_t i = 0; i < write_files.size(); ++i) { - auto writer = getWriter(write_files[i], store_paths[i % store_paths.size()]); + auto writer = checkAndRenewWriter(write_files[i], /*parent_path_hint=*/store_paths[i % store_paths.size()]); idle_writers.emplace_back(std::move(writer)); } #endif @@ -347,9 +347,13 @@ PageEntry PageStorage::getEntry(PageId page_id, SnapshotPtr snapshot) } } -PageStorage::WriterPtr PageStorage::getWriter( // +// Check whether `page_file` is writable or not, renew `page_file` if need and return its belonging writer. +// - Writable, reuse `old_writer` if it is not a nullptr, otherwise, create a new writer from `page_file` +// - Not writable, renew the `page_file` and its belonging writer. +// The of the new `page_file` is of all `write_files` +PageStorage::WriterPtr PageStorage::checkAndRenewWriter( // PageFile & page_file, - const String & pf_parent_path_hint, + const String & parent_path_hint, PageStorage::WriterPtr && old_writer, const String & logging_msg) { @@ -360,22 +364,33 @@ PageStorage::WriterPtr PageStorage::getWriter( // if (is_writable) { if (old_writer) + { + // Reuse the old_writer instead of creating a new writer write_file_writer.swap(old_writer); + } else + { + // Create a new writer from `page_file` write_file_writer = page_file.createWriter(config.sync_on_write, false); + } } else { - String pf_parent_path = delegator->normalPath(); + /// Create a new PageFile and generate its belonging writer + + String pf_parent_path = delegator->defaultPath(); if (old_writer) { + // If the old_writer is not NULL, use the same parent path to create a new writer. + // So the number of writers on different paths keep unchanged. pf_parent_path = old_writer->parentPath(); // Reset writer to ensure all data have been flushed. old_writer.reset(); } - else if (!pf_parent_path_hint.empty()) + else if (!parent_path_hint.empty()) { - pf_parent_path = pf_parent_path_hint; + // Check whether caller has defined a hint path + pf_parent_path = parent_path_hint; } PageFileIdAndLevel max_writing_id_lvl{0, 0}; @@ -452,8 +467,8 @@ void PageStorage::write(WriteBatch && wb) } } auto & page_file = write_files[index]; - file_to_write - = getWriter(page_file, "", std::move(file_to_write), " PageFile_" + DB::toString(page_file.getFileId()) + "_0 is full,"); + file_to_write = checkAndRenewWriter( + page_file, "", std::move(file_to_write), /*logging_msg=*/" PageFile_" + DB::toString(page_file.getFileId()) + "_0 is full,"); idle_writers.emplace_back(std::move(file_to_write)); @@ -936,7 +951,7 @@ void PageStorage::archivePageFiles(const PageFileSet & page_files) if (page_files.empty()) return; - const Poco::Path archive_path(delegator->normalPath(), PageStorage::ARCHIVE_SUBDIR); + const Poco::Path archive_path(delegator->defaultPath(), PageStorage::ARCHIVE_SUBDIR); Poco::File archive_dir(archive_path); if (!archive_dir.exists()) archive_dir.createDirectory(); diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 643dd36f308..05ea1e96394 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -156,10 +156,10 @@ class PageStorage const ListPageFilesOption & option = ListPageFilesOption()); private: - WriterPtr getWriter(PageFile & page_file, - const String & pf_parent_path_hint = "", - WriterPtr && old_writer = nullptr, - const String & logging_msg = ""); + WriterPtr checkAndRenewWriter(PageFile & page_file, + const String & parent_path_hint, + WriterPtr && old_writer = nullptr, + const String & logging_msg = ""); ReaderPtr getReader(const PageFileIdAndLevel & file_id_level); static constexpr const char * ARCHIVE_SUBDIR = "archive"; diff --git a/dbms/src/Storages/Page/gc/LegacyCompactor.cpp b/dbms/src/Storages/Page/gc/LegacyCompactor.cpp index 0dc42bd5230..025844aaad5 100644 --- a/dbms/src/Storages/Page/gc/LegacyCompactor.cpp +++ b/dbms/src/Storages/Page/gc/LegacyCompactor.cpp @@ -45,7 +45,7 @@ LegacyCompactor::tryCompact( // // Use the largest id-level in page_files_to_compact as Checkpoint's file const PageFileIdAndLevel checkpoint_id = page_files_to_compact.rbegin()->fileIdLevel(); - const String storage_path = delegator->normalPath(); + const String storage_path = delegator->defaultPath(); if (PageFile::isPageFileExist(checkpoint_id, storage_path, file_provider, PageFile::Type::Checkpoint, page_file_log)) { LOG_WARNING(log, diff --git a/dbms/src/Storages/Page/tests/gtest_data_compactor.cpp b/dbms/src/Storages/Page/tests/gtest_data_compactor.cpp index 0d3e5fb545e..24667719505 100644 --- a/dbms/src/Storages/Page/tests/gtest_data_compactor.cpp +++ b/dbms/src/Storages/Page/tests/gtest_data_compactor.cpp @@ -38,7 +38,7 @@ try auto & ctx = TiFlashTestEnv::getContext(DB::Settings(), test_paths); const FileProviderPtr file_provider = ctx.getFileProvider(); auto pool = ctx.getExtraPaths().withTable("test", "t", false); - auto delegate = pool.getMultiDiskDelegate("log"); + auto delegate = pool.getPSDiskDelegatorMulti("log"); PageStorage storage("data_compact_test", delegate, config, file_provider); diff --git a/dbms/src/Storages/Page/tests/gtest_legacy_compactor.cpp b/dbms/src/Storages/Page/tests/gtest_legacy_compactor.cpp index ea53e8f5e6a..acdb2e2155c 100644 --- a/dbms/src/Storages/Page/tests/gtest_legacy_compactor.cpp +++ b/dbms/src/Storages/Page/tests/gtest_legacy_compactor.cpp @@ -168,7 +168,7 @@ try auto & ctx = TiFlashTestEnv::getContext(); const FileProviderPtr file_provider = ctx.getFileProvider(); StoragePathPool spool = ctx.getExtraPaths().withTable("test", "t", false); - auto delegator = spool.getNormalDiskDelegate("meta"); + auto delegator = spool.getPSDiskDelegatorSingle("meta"); PageStorage storage("compact_test", delegator, PageStorage::Config{}, file_provider); PageStorage::ListPageFilesOption opt; @@ -185,7 +185,7 @@ try // TODO: PageFile page_file - = PageFile::openPageFileForRead(7, 0, delegator->normalPath(), file_provider, PageFile::Type::Checkpoint, storage.page_file_log); + = PageFile::openPageFileForRead(7, 0, delegator->defaultPath(), file_provider, PageFile::Type::Checkpoint, storage.page_file_log); ASSERT_TRUE(page_file.isExist()); PageStorage::MetaMergingQueue mergine_queue; diff --git a/dbms/src/Storages/Page/tests/gtest_page_storage.cpp b/dbms/src/Storages/Page/tests/gtest_page_storage.cpp index 0f98be1d302..5f1f19e5221 100644 --- a/dbms/src/Storages/Page/tests/gtest_page_storage.cpp +++ b/dbms/src/Storages/Page/tests/gtest_page_storage.cpp @@ -45,7 +45,7 @@ class PageStorage_test : public ::testing::Test // drop dir if exists auto & ctx = TiFlashTestEnv::getContext(); path_pool = std::make_unique(ctx.getExtraPaths().withTable("test", "t1", false)); - for (const auto & p : path_pool->getNormalDiskDelegate("log")->listPaths()) + for (const auto & p : path_pool->getPSDiskDelegatorSingle("log")->listPaths()) { if (Poco::File file(p); file.exists()) { @@ -62,7 +62,7 @@ class PageStorage_test : public ::testing::Test std::shared_ptr reopenWithConfig(const PageStorage::Config & config_) { auto & ctx = TiFlashTestEnv::getContext(); - auto delegator = path_pool->getNormalDiskDelegate("log"); + auto delegator = path_pool->getPSDiskDelegatorSingle("log"); auto storage = std::make_shared("test.t", delegator, config_, file_provider, ctx.getTiFlashMetrics()); storage->restore(); return storage; @@ -382,7 +382,7 @@ try wb.putPage(1, 0, buf, buf_sz); storage->write(std::move(wb)); - auto f = PageFile::openPageFileForRead(1, 0, storage->delegator->normalPath(), file_provider, PageFile::Type::Formal, storage->log); + auto f = PageFile::openPageFileForRead(1, 0, storage->delegator->defaultPath(), file_provider, PageFile::Type::Formal, storage->log); f.setLegacy(); } @@ -393,7 +393,7 @@ try auto buf = std::make_shared(c_buff, sizeof(c_buff)); wb.putPage(1, 0, buf, buf_sz); - auto f = PageFile::newPageFile(2, 0, storage->delegator->normalPath(), file_provider, PageFile::Type::Temp, storage->log); + auto f = PageFile::newPageFile(2, 0, storage->delegator->defaultPath(), file_provider, PageFile::Type::Temp, storage->log); { auto w = f.createWriter(false, true); diff --git a/dbms/src/Storages/Page/tests/gtest_page_storage_multi_paths.cpp b/dbms/src/Storages/Page/tests/gtest_page_storage_multi_paths.cpp index 4561f1745a9..95e9cd132d1 100644 --- a/dbms/src/Storages/Page/tests/gtest_page_storage_multi_paths.cpp +++ b/dbms/src/Storages/Page/tests/gtest_page_storage_multi_paths.cpp @@ -94,7 +94,7 @@ try auto capacity = std::make_shared(all_paths, 0); StoragePathPool pool = PathPool(all_paths, all_paths, capacity, file_provider).withTable("test", "table", false); - storage = std::make_shared("test.table", pool.getMultiDiskDelegate("log"), config, file_provider); + storage = std::make_shared("test.table", pool.getPSDiskDelegatorMulti("log"), config, file_provider); storage->restore(); const UInt64 tag = 0; @@ -132,7 +132,7 @@ try } // restore - storage = std::make_shared("test.t", pool.getMultiDiskDelegate("log"), config, file_provider); + storage = std::make_shared("test.t", pool.getPSDiskDelegatorMulti("log"), config, file_provider); storage->restore(); // Read again @@ -188,7 +188,7 @@ try } // Restore. This ensure last write is correct. - storage = std::make_shared("test.t", pool.getMultiDiskDelegate("log"), config, file_provider); + storage = std::make_shared("test.t", pool.getPSDiskDelegatorMulti("log"), config, file_provider); storage->restore(); // Read again to check all data. diff --git a/dbms/src/Storages/Page/tests/gtest_page_storage_multi_writers.cpp b/dbms/src/Storages/Page/tests/gtest_page_storage_multi_writers.cpp index e64bd715a59..611ba46e854 100644 --- a/dbms/src/Storages/Page/tests/gtest_page_storage_multi_writers.cpp +++ b/dbms/src/Storages/Page/tests/gtest_page_storage_multi_writers.cpp @@ -66,7 +66,7 @@ class PageStorageMultiWriters_test : public ::testing::Test std::shared_ptr reopenWithConfig(const PageStorage::Config & config_) { auto spool = TiFlashTestEnv::getContext().getExtraPaths().withTable("test", "t", false); - auto delegator = spool.getNormalDiskDelegate("log"); + auto delegator = spool.getPSDiskDelegatorSingle("log"); auto storage = std::make_shared("test.t", delegator, config_, file_provider); storage->restore(); return storage; diff --git a/dbms/src/Storages/PathPool.cpp b/dbms/src/Storages/PathPool.cpp index bdf9854f890..836db3cb44a 100644 --- a/dbms/src/Storages/PathPool.cpp +++ b/dbms/src/Storages/PathPool.cpp @@ -237,7 +237,7 @@ void StoragePathPool::renamePath(const String & old_path, const String & new_pat // Stable data //========================================================================================== -Strings StableDelegator::listPaths() const +Strings StableDiskDelegator::listPaths() const { std::vector paths; for (size_t i = 0; i < pool.main_path_infos.size(); ++i) @@ -247,7 +247,7 @@ Strings StableDelegator::listPaths() const return paths; } -String StableDelegator::choosePath() const +String StableDiskDelegator::choosePath() const { std::lock_guard lock{pool.mutex}; UInt64 total_size = 0; @@ -278,7 +278,7 @@ String StableDelegator::choosePath() const throw Exception("Should not reach here", ErrorCodes::LOGICAL_ERROR); } -String StableDelegator::getDTFilePath(UInt64 file_id) const +String StableDiskDelegator::getDTFilePath(UInt64 file_id) const { std::lock_guard lock{pool.mutex}; auto iter = pool.dt_file_path_map.find(file_id); @@ -287,7 +287,7 @@ String StableDelegator::getDTFilePath(UInt64 file_id) const throw Exception("Can not find path for DMFile [id=" + toString(file_id) + "]"); } -void StableDelegator::addDTFile(UInt64 file_id, size_t file_size, std::string_view path) +void StableDiskDelegator::addDTFile(UInt64 file_id, size_t file_size, std::string_view path) { path.remove_suffix(1 + strlen(StoragePathPool::STABLE_FOLDER_NAME)); // remove '/stable' added in listPathsForStable/getDTFilePath std::lock_guard lock{pool.mutex}; @@ -316,7 +316,7 @@ void StableDelegator::addDTFile(UInt64 file_id, size_t file_size, std::string_vi pool.global_capacity->addUsedSize(path, file_size); } -void StableDelegator::removeDTFile(UInt64 file_id) +void StableDiskDelegator::removeDTFile(UInt64 file_id) { std::lock_guard lock{pool.mutex}; auto iter = pool.dt_file_path_map.find(file_id); @@ -335,11 +335,11 @@ void StableDelegator::removeDTFile(UInt64 file_id) // Delta data //========================================================================================== -size_t MultiDiskDelegator::numPaths() const { return pool.latest_path_infos.size(); } +size_t PSDiskDelegatorMulti::numPaths() const { return pool.latest_path_infos.size(); } -String MultiDiskDelegator::normalPath() const { return pool.latest_path_infos[0].path + "/" + path_prefix; } +String PSDiskDelegatorMulti::defaultPath() const { return pool.latest_path_infos[0].path + "/" + path_prefix; } -Strings MultiDiskDelegator::listPaths() const +Strings PSDiskDelegatorMulti::listPaths() const { // The delta data could be stored in all direcotries. std::vector paths; @@ -350,7 +350,7 @@ Strings MultiDiskDelegator::listPaths() const return paths; } -String MultiDiskDelegator::choosePath(const PageFileIdAndLevel & id_lvl) +String PSDiskDelegatorMulti::choosePath(const PageFileIdAndLevel & id_lvl) { auto return_path = [&](const size_t index) -> String { return pool.latest_path_infos[index].path + "/" + path_prefix; }; @@ -392,7 +392,7 @@ String MultiDiskDelegator::choosePath(const PageFileIdAndLevel & id_lvl) throw Exception("Should not reach here", ErrorCodes::LOGICAL_ERROR); } -size_t MultiDiskDelegator::addPageFileUsedSize( +size_t PSDiskDelegatorMulti::addPageFileUsedSize( const PageFileIdAndLevel & id_lvl, size_t size_to_add, const String & pf_parent_path, bool need_insert_location) { // Get a normalized path without `path_prefix` and trailing '/' @@ -423,7 +423,7 @@ size_t MultiDiskDelegator::addPageFileUsedSize( return index; } -String MultiDiskDelegator::getPageFilePath(const PageFileIdAndLevel & id_lvl) const +String PSDiskDelegatorMulti::getPageFilePath(const PageFileIdAndLevel & id_lvl) const { std::lock_guard lock{pool.mutex}; auto iter = page_path_map.find(id_lvl); @@ -432,7 +432,7 @@ String MultiDiskDelegator::getPageFilePath(const PageFileIdAndLevel & id_lvl) co throw Exception("Can not find path for PageFile [id=" + toString(id_lvl.first) + "_" + toString(id_lvl.second) + "]"); } -void MultiDiskDelegator::removePageFile(const PageFileIdAndLevel & id_lvl, size_t file_size) +void PSDiskDelegatorMulti::removePageFile(const PageFileIdAndLevel & id_lvl, size_t file_size) { std::lock_guard lock{pool.mutex}; auto iter = page_path_map.find(id_lvl); @@ -449,11 +449,11 @@ void MultiDiskDelegator::removePageFile(const PageFileIdAndLevel & id_lvl, size_ // Normal data //========================================================================================== -size_t NormalDiskDelegator::numPaths() const { return 1; } +size_t PSDiskDelegatorSingle::numPaths() const { return 1; } -String NormalDiskDelegator::normalPath() const { return pool.latest_path_infos[0].path + "/" + path_prefix; } +String PSDiskDelegatorSingle::defaultPath() const { return pool.latest_path_infos[0].path + "/" + path_prefix; } -Strings NormalDiskDelegator::listPaths() const +Strings PSDiskDelegatorSingle::listPaths() const { // only stored in the first path. std::vector paths; @@ -461,9 +461,9 @@ Strings NormalDiskDelegator::listPaths() const return paths; } -String NormalDiskDelegator::choosePath(const PageFileIdAndLevel & /*id_lvl*/) { return pool.latest_path_infos[0].path + "/" + path_prefix; } +String PSDiskDelegatorSingle::choosePath(const PageFileIdAndLevel & /*id_lvl*/) { return pool.latest_path_infos[0].path + "/" + path_prefix; } -size_t NormalDiskDelegator::addPageFileUsedSize( +size_t PSDiskDelegatorSingle::addPageFileUsedSize( const PageFileIdAndLevel & /*id_lvl*/, size_t size_to_add, const String & pf_parent_path, bool /*need_insert_location*/) { // In this case, inserting to page_path_map or adding total_size for PathInfo seems useless. @@ -472,12 +472,12 @@ size_t NormalDiskDelegator::addPageFileUsedSize( return 0; } -String NormalDiskDelegator::getPageFilePath(const PageFileIdAndLevel & /*id_lvl*/) const +String PSDiskDelegatorSingle::getPageFilePath(const PageFileIdAndLevel & /*id_lvl*/) const { return pool.latest_path_infos[0].path + "/" + path_prefix; } -void NormalDiskDelegator::removePageFile(const PageFileIdAndLevel & /*id_lvl*/, size_t file_size) +void PSDiskDelegatorSingle::removePageFile(const PageFileIdAndLevel & /*id_lvl*/, size_t file_size) { pool.global_capacity->freeUsedSize(pool.latest_path_infos[0].path, file_size); } diff --git a/dbms/src/Storages/PathPool.h b/dbms/src/Storages/PathPool.h index ba8acb4a63d..0419f959ffd 100644 --- a/dbms/src/Storages/PathPool.h +++ b/dbms/src/Storages/PathPool.h @@ -23,14 +23,16 @@ class PathPool; /// A class to manage paths for the specified storage. class StoragePathPool; -/// Delegators to StoragePathPool. They are used for managing the path for storing stable/delta/raft data. -class StableDelegator; +/// ===== Delegators to StoragePathPool ===== /// +/// Delegators to StoragePathPool. Use for managing the path of DTFiles. +class StableDiskDelegator; +/// Delegators to StoragePathPool. Use by PageStorage for managing the path of PageFiles. class PSDiskDelegator; using PSDiskDelegatorPtr = std::shared_ptr; -class MultiDiskDelegator; -class NormalDiskDelegator; -// TODO: support multi-paths for RaftDelegator -// using RaftDelegator = NormalDiskDelegator; +class PSDiskDelegatorMulti; +class PSDiskDelegatorSingle; +// TODO: support multi-paths for Raft data +// using PSDiskDelegatorRaft = PSDiskDelegatorSingle; /// A class to manage global paths. @@ -70,10 +72,10 @@ class PathPool Poco::Logger * log; }; -class StableDelegator : private boost::noncopyable +class StableDiskDelegator : private boost::noncopyable { public: - StableDelegator(StoragePathPool & pool_) : pool(pool_) {} + StableDiskDelegator(StoragePathPool & pool_) : pool(pool_) {} Strings listPaths() const; @@ -92,12 +94,11 @@ class StableDelegator : private boost::noncopyable class PSDiskDelegator : private boost::noncopyable { public: - PSDiskDelegator() {} virtual ~PSDiskDelegator() {} virtual size_t numPaths() const = 0; - virtual String normalPath() const = 0; + virtual String defaultPath() const = 0; virtual Strings listPaths() const = 0; @@ -112,14 +113,14 @@ class PSDiskDelegator : private boost::noncopyable virtual void removePageFile(const PageFileIdAndLevel & id_lvl, size_t file_size) = 0; }; -class MultiDiskDelegator : public PSDiskDelegator +class PSDiskDelegatorMulti : public PSDiskDelegator { public: - MultiDiskDelegator(StoragePathPool & pool_, String prefix) : pool(pool_), path_prefix(std::move(prefix)) {} + PSDiskDelegatorMulti(StoragePathPool & pool_, String prefix) : pool(pool_), path_prefix(std::move(prefix)) {} size_t numPaths() const override; - String normalPath() const override; + String defaultPath() const override; Strings listPaths() const override; @@ -139,14 +140,14 @@ class MultiDiskDelegator : public PSDiskDelegator PathPool::PageFilePathMap page_path_map; }; -class NormalDiskDelegator : public PSDiskDelegator +class PSDiskDelegatorSingle : public PSDiskDelegator { public: - NormalDiskDelegator(StoragePathPool & pool_, String prefix) : pool(pool_), path_prefix(std::move(prefix)) {} + PSDiskDelegatorSingle(StoragePathPool & pool_, String prefix) : pool(pool_), path_prefix(std::move(prefix)) {} size_t numPaths() const override; - String normalPath() const override; + String defaultPath() const override; Strings listPaths() const override; @@ -179,16 +180,16 @@ class StoragePathPool // Generate a lightweight delegator for managing stable data, such as choosing path for DTFile or getting DTFile path by ID and so on. // Those paths are generated from `main_path_infos` and `STABLE_FOLDER_NAME` - StableDelegator getStableDelegate() { return StableDelegator(*this); } + StableDiskDelegator getStableDiskDelegator() { return StableDiskDelegator(*this); } // Generate a delegator for managing the paths of `StoragePool`. // Those paths are generated from `latest_path_infos` and `prefix`. // User should keep the pointer to track the PageFileID -> path index mapping. - PSDiskDelegatorPtr getMultiDiskDelegate(const String & prefix) { return std::make_shared(*this, prefix); } + PSDiskDelegatorPtr getPSDiskDelegatorMulti(const String & prefix) { return std::make_shared(*this, prefix); } // Generate a delegator for managing the paths of `StoragePool`. // Those paths are generated from the first path of `latest_path_infos` and `prefix` - PSDiskDelegatorPtr getNormalDiskDelegate(const String & prefix) { return std::make_shared(*this, prefix); } + PSDiskDelegatorPtr getPSDiskDelegatorSingle(const String & prefix) { return std::make_shared(*this, prefix); } void rename(const String & new_database, const String & new_table, bool clean_rename); @@ -215,9 +216,9 @@ class StoragePathPool }; using LatestPathInfos = std::vector; - friend class StableDelegator; - friend class MultiDiskDelegator; - friend class NormalDiskDelegator; + friend class StableDiskDelegator; + friend class PSDiskDelegatorMulti; + friend class PSDiskDelegatorSingle; private: // Path, size diff --git a/dbms/src/Storages/tests/gtest_path_pool.cpp b/dbms/src/Storages/tests/gtest_path_pool.cpp index 681941052e5..cb1edec2f7a 100644 --- a/dbms/src/Storages/tests/gtest_path_pool.cpp +++ b/dbms/src/Storages/tests/gtest_path_pool.cpp @@ -39,7 +39,7 @@ try // Stable delegate { - auto delegate = spool.getStableDelegate(); + auto delegate = spool.getStableDiskDelegator(); auto res = delegate.listPaths(); EXPECT_EQ(res.size(), paths.size()); for (size_t i = 0; i < res.size(); ++i) @@ -62,7 +62,7 @@ try } // Delta delegate { - auto delegate = spool.getMultiDiskDelegate("log"); + auto delegate = spool.getPSDiskDelegatorMulti("log"); auto res = delegate->listPaths(); EXPECT_EQ(res.size(), paths.size()); for (size_t i = 0; i < res.size(); ++i) @@ -89,7 +89,7 @@ try } // Normal delegate { - auto delegate = spool.getNormalDiskDelegate("meta"); + auto delegate = spool.getPSDiskDelegatorSingle("meta"); auto res = delegate->listPaths(); EXPECT_EQ(res.size(), 1UL); for (size_t i = 0; i < res.size(); ++i) @@ -128,7 +128,7 @@ try auto spool = pool.withTable("test", "t", false); // Stable delegate { - auto delegate = spool.getStableDelegate(); + auto delegate = spool.getStableDiskDelegator(); auto res = delegate.listPaths(); EXPECT_EQ(res.size(), paths.size()); for (size_t i = 0; i < res.size(); ++i) @@ -151,7 +151,7 @@ try } // Delta delegate { - auto delegate = spool.getMultiDiskDelegate("log"); + auto delegate = spool.getPSDiskDelegatorMulti("log"); auto res = delegate->listPaths(); EXPECT_EQ(res.size(), latest_paths.size()); for (size_t i = 0; i < res.size(); ++i) @@ -178,7 +178,7 @@ try } // Normal delegate { - auto delegate = spool.getNormalDiskDelegate("meta"); + auto delegate = spool.getPSDiskDelegatorSingle("meta"); auto res = delegate->listPaths(); EXPECT_EQ(res.size(), 1UL); for (size_t i = 0; i < res.size(); ++i)