diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index dfaf5b24943..d416e5a6598 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -179,7 +179,7 @@ void AsynchronousMetrics::update() { if (auto store = dt_storage->getStoreIfInited(); store) { - auto stat = store->getStat(); + auto stat = store->getStoreStats(); calculateMax(max_dt_stable_oldest_snapshot_lifetime, stat.storage_stable_oldest_snapshot_lifetime); calculateMax(max_dt_delta_oldest_snapshot_lifetime, stat.storage_delta_oldest_snapshot_lifetime); calculateMax(max_dt_meta_oldest_snapshot_lifetime, stat.storage_meta_oldest_snapshot_lifetime); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index d618cfa0df6..3e16fcf0d67 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -1492,206 +1492,6 @@ void DeltaMergeStore::restoreStableFiles() } } -static inline DB::PS::V2::PageEntriesVersionSetWithDelta::Snapshot * -toConcreteSnapshot(const DB::PageStorage::SnapshotPtr & ptr) -{ - return dynamic_cast(ptr.get()); -} - -DeltaMergeStoreStat DeltaMergeStore::getStat() -{ - std::shared_lock lock(read_write_mutex); - - DeltaMergeStoreStat stat; - - if (shutdown_called.load(std::memory_order_relaxed)) - return stat; - - stat.segment_count = segments.size(); - - Int64 total_placed_rows = 0; - Int64 total_delta_cache_rows = 0; - Float64 total_delta_cache_size = 0; - Int64 total_delta_valid_cache_rows = 0; - for (const auto & [handle, segment] : segments) - { - (void)handle; - const auto & delta = segment->getDelta(); - const auto & stable = segment->getStable(); - - total_placed_rows += delta->getPlacedDeltaRows(); - - if (delta->getColumnFileCount()) - { - stat.total_rows += delta->getRows(); - stat.total_size += delta->getBytes(); - - stat.total_delete_ranges += delta->getDeletes(); - - stat.delta_count += 1; - stat.total_pack_count_in_delta += delta->getColumnFileCount(); - - stat.total_delta_rows += delta->getRows(); - stat.total_delta_size += delta->getBytes(); - - stat.delta_index_size += delta->getDeltaIndexBytes(); - - total_delta_cache_rows += delta->getTotalCacheRows(); - total_delta_cache_size += delta->getTotalCacheBytes(); - total_delta_valid_cache_rows += delta->getValidCacheRows(); - } - - if (stable->getDMFilesPacks()) - { - stat.total_rows += stable->getRows(); - stat.total_size += stable->getBytes(); - - stat.stable_count += 1; - stat.total_pack_count_in_stable += stable->getDMFilesPacks(); - - stat.total_stable_rows += stable->getRows(); - stat.total_stable_size += stable->getBytes(); - stat.total_stable_size_on_disk += stable->getDMFilesBytesOnDisk(); - } - } - - stat.delta_rate_rows = static_cast(stat.total_delta_rows) / stat.total_rows; - stat.delta_rate_segments = static_cast(stat.delta_count) / stat.segment_count; - - stat.delta_placed_rate = static_cast(total_placed_rows) / stat.total_delta_rows; - stat.delta_cache_size = total_delta_cache_size; - stat.delta_cache_rate = static_cast(total_delta_valid_cache_rows) / stat.total_delta_rows; - stat.delta_cache_wasted_rate = static_cast(total_delta_cache_rows - total_delta_valid_cache_rows) / total_delta_valid_cache_rows; - - stat.avg_segment_rows = static_cast(stat.total_rows) / stat.segment_count; - stat.avg_segment_size = static_cast(stat.total_size) / stat.segment_count; - - stat.avg_delta_rows = static_cast(stat.total_delta_rows) / stat.delta_count; - stat.avg_delta_size = static_cast(stat.total_delta_size) / stat.delta_count; - stat.avg_delta_delete_ranges = static_cast(stat.total_delete_ranges) / stat.delta_count; - - stat.avg_stable_rows = static_cast(stat.total_stable_rows) / stat.stable_count; - stat.avg_stable_size = static_cast(stat.total_stable_size) / stat.stable_count; - - stat.avg_pack_count_in_delta = static_cast(stat.total_pack_count_in_delta) / stat.delta_count; - stat.avg_pack_rows_in_delta = static_cast(stat.total_delta_rows) / stat.total_pack_count_in_delta; - stat.avg_pack_size_in_delta = static_cast(stat.total_delta_size) / stat.total_pack_count_in_delta; - - stat.avg_pack_count_in_stable = static_cast(stat.total_pack_count_in_stable) / stat.stable_count; - stat.avg_pack_rows_in_stable = static_cast(stat.total_stable_rows) / stat.total_pack_count_in_stable; - stat.avg_pack_size_in_stable = static_cast(stat.total_stable_size) / stat.total_pack_count_in_stable; - - static const String useless_tracing_id("DeltaMergeStore::getStat"); - { - auto snaps_stat = storage_pool->dataReader()->getSnapshotsStat(); - stat.storage_stable_num_snapshots = snaps_stat.num_snapshots; - stat.storage_stable_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; - stat.storage_stable_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; - stat.storage_stable_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - PageStorage::SnapshotPtr stable_snapshot = storage_pool->dataReader()->getSnapshot(useless_tracing_id); - const auto * concrete_snap = toConcreteSnapshot(stable_snapshot); - if (concrete_snap) - { - if (const auto * const version = concrete_snap->version(); version != nullptr) - { - stat.storage_stable_num_pages = version->numPages(); - stat.storage_stable_num_normal_pages = version->numNormalPages(); - stat.storage_stable_max_page_id = version->maxId(); - } - else - { - LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=data"); - } - } - } - { - auto snaps_stat = storage_pool->logReader()->getSnapshotsStat(); - stat.storage_delta_num_snapshots = snaps_stat.num_snapshots; - stat.storage_delta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; - stat.storage_delta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; - stat.storage_delta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - PageStorage::SnapshotPtr log_snapshot = storage_pool->logReader()->getSnapshot(useless_tracing_id); - const auto * concrete_snap = toConcreteSnapshot(log_snapshot); - if (concrete_snap) - { - if (const auto * const version = concrete_snap->version(); version != nullptr) - { - stat.storage_delta_num_pages = version->numPages(); - stat.storage_delta_num_normal_pages = version->numNormalPages(); - stat.storage_delta_max_page_id = version->maxId(); - } - else - { - LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=log"); - } - } - } - { - auto snaps_stat = storage_pool->metaReader()->getSnapshotsStat(); - stat.storage_meta_num_snapshots = snaps_stat.num_snapshots; - stat.storage_meta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; - stat.storage_meta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; - stat.storage_meta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - PageStorage::SnapshotPtr meta_snapshot = storage_pool->metaReader()->getSnapshot(useless_tracing_id); - const auto * concrete_snap = toConcreteSnapshot(meta_snapshot); - if (concrete_snap) - { - if (const auto * const version = concrete_snap->version(); version != nullptr) - { - stat.storage_meta_num_pages = version->numPages(); - stat.storage_meta_num_normal_pages = version->numNormalPages(); - stat.storage_meta_max_page_id = version->maxId(); - } - else - { - LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=meta"); - } - } - } - - stat.background_tasks_length = background_tasks.length(); - - return stat; -} - -SegmentStats DeltaMergeStore::getSegmentStats() -{ - std::shared_lock lock(read_write_mutex); - - SegmentStats stats; - for (const auto & [handle, segment] : segments) - { - (void)handle; - - SegmentStat stat; - const auto & delta = segment->getDelta(); - const auto & stable = segment->getStable(); - - stat.segment_id = segment->segmentId(); - stat.range = segment->getRowKeyRange(); - - stat.rows = segment->getEstimatedRows(); - stat.size = delta->getBytes() + stable->getBytes(); - stat.delete_ranges = delta->getDeletes(); - - stat.stable_size_on_disk = stable->getDMFilesBytesOnDisk(); - - stat.delta_pack_count = delta->getColumnFileCount(); - stat.stable_pack_count = stable->getDMFilesPacks(); - - stat.avg_delta_pack_rows = static_cast(delta->getRows()) / stat.delta_pack_count; - stat.avg_stable_pack_rows = static_cast(stable->getRows()) / stat.stable_pack_count; - - stat.delta_rate = static_cast(delta->getRows()) / stat.rows; - stat.delta_cache_size = delta->getTotalCacheBytes(); - - stat.delta_index_size = delta->getDeltaIndexBytes(); - - stats.push_back(stat); - } - return stats; -} - SegmentReadTasks DeltaMergeStore::getReadTasksByRanges( DMContext & dm_context, const RowKeyRanges & sorted_ranges, diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index 0e81b738342..d934d455f47 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -52,31 +52,40 @@ struct ExternalDTFileInfo; inline static const PageId DELTA_MERGE_FIRST_SEGMENT_ID = 1; -struct SegmentStat +struct SegmentStats { UInt64 segment_id = 0; RowKeyRange range; - + UInt64 epoch = 0; UInt64 rows = 0; UInt64 size = 0; - UInt64 delete_ranges = 0; - - UInt64 stable_size_on_disk = 0; - - UInt64 delta_pack_count = 0; - UInt64 stable_pack_count = 0; - - Float64 avg_delta_pack_rows = 0; - Float64 avg_stable_pack_rows = 0; Float64 delta_rate = 0; + UInt64 delta_memtable_rows = 0; + UInt64 delta_memtable_size = 0; + UInt64 delta_memtable_column_files = 0; + UInt64 delta_memtable_delete_ranges = 0; + UInt64 delta_persisted_page_id = 0; + UInt64 delta_persisted_rows = 0; + UInt64 delta_persisted_size = 0; + UInt64 delta_persisted_column_files = 0; + UInt64 delta_persisted_delete_ranges = 0; UInt64 delta_cache_size = 0; - UInt64 delta_index_size = 0; + + UInt64 stable_page_id = 0; + UInt64 stable_rows = 0; + UInt64 stable_size = 0; + UInt64 stable_dmfiles = 0; + UInt64 stable_dmfiles_id_0 = 0; + UInt64 stable_dmfiles_rows = 0; + UInt64 stable_dmfiles_size = 0; + UInt64 stable_dmfiles_size_on_disk = 0; + UInt64 stable_dmfiles_packs = 0; }; -using SegmentStats = std::vector; +using SegmentsStats = std::vector; -struct DeltaMergeStoreStat +struct StoreStats { UInt64 segment_count = 0; @@ -385,16 +394,12 @@ class DeltaMergeStore : private boost::noncopyable SortDescription getPrimarySortDescription() const; void check(const Context & db_context); - DeltaMergeStoreStat getStat(); - SegmentStats getSegmentStats(); - bool isCommonHandle() const - { - return is_common_handle; - } - size_t getRowKeyColumnSize() const - { - return rowkey_column_size; - } + + StoreStats getStoreStats(); + SegmentsStats getSegmentsStats(); + + bool isCommonHandle() const { return is_common_handle; } + size_t getRowKeyColumnSize() const { return rowkey_column_size; } public: /// Methods mainly used by region split. diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp new file mode 100644 index 00000000000..507cd539ba2 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp @@ -0,0 +1,236 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +namespace DB +{ +namespace DM +{ + +static inline auto * +toConcreteSnapshot(const DB::PageStorage::SnapshotPtr & ptr) +{ + return dynamic_cast(ptr.get()); +} + +StoreStats DeltaMergeStore::getStoreStats() +{ + std::shared_lock lock(read_write_mutex); + + StoreStats stat; + + if (shutdown_called.load(std::memory_order_relaxed)) + return stat; + + stat.segment_count = segments.size(); + + Int64 total_placed_rows = 0; + Int64 total_delta_cache_rows = 0; + Float64 total_delta_cache_size = 0; + Int64 total_delta_valid_cache_rows = 0; + for (const auto & [handle, segment] : segments) + { + UNUSED(handle); + const auto & delta = segment->getDelta(); + const auto & stable = segment->getStable(); + + total_placed_rows += delta->getPlacedDeltaRows(); + + if (delta->getColumnFileCount()) + { + stat.total_rows += delta->getRows(); + stat.total_size += delta->getBytes(); + + stat.total_delete_ranges += delta->getDeletes(); + + stat.delta_count += 1; + stat.total_pack_count_in_delta += delta->getColumnFileCount(); + + stat.total_delta_rows += delta->getRows(); + stat.total_delta_size += delta->getBytes(); + + stat.delta_index_size += delta->getDeltaIndexBytes(); + + total_delta_cache_rows += delta->getTotalCacheRows(); + total_delta_cache_size += delta->getTotalCacheBytes(); + total_delta_valid_cache_rows += delta->getValidCacheRows(); + } + + if (stable->getDMFilesPacks()) + { + stat.total_rows += stable->getRows(); + stat.total_size += stable->getBytes(); + + stat.stable_count += 1; + stat.total_pack_count_in_stable += stable->getDMFilesPacks(); + + stat.total_stable_rows += stable->getRows(); + stat.total_stable_size += stable->getBytes(); + stat.total_stable_size_on_disk += stable->getDMFilesBytesOnDisk(); + } + } + + stat.delta_rate_rows = static_cast(stat.total_delta_rows) / stat.total_rows; + stat.delta_rate_segments = static_cast(stat.delta_count) / stat.segment_count; + + stat.delta_placed_rate = static_cast(total_placed_rows) / stat.total_delta_rows; + stat.delta_cache_size = total_delta_cache_size; + stat.delta_cache_rate = static_cast(total_delta_valid_cache_rows) / stat.total_delta_rows; + stat.delta_cache_wasted_rate = static_cast(total_delta_cache_rows - total_delta_valid_cache_rows) / total_delta_valid_cache_rows; + + stat.avg_segment_rows = static_cast(stat.total_rows) / stat.segment_count; + stat.avg_segment_size = static_cast(stat.total_size) / stat.segment_count; + + stat.avg_delta_rows = static_cast(stat.total_delta_rows) / stat.delta_count; + stat.avg_delta_size = static_cast(stat.total_delta_size) / stat.delta_count; + stat.avg_delta_delete_ranges = static_cast(stat.total_delete_ranges) / stat.delta_count; + + stat.avg_stable_rows = static_cast(stat.total_stable_rows) / stat.stable_count; + stat.avg_stable_size = static_cast(stat.total_stable_size) / stat.stable_count; + + stat.avg_pack_count_in_delta = static_cast(stat.total_pack_count_in_delta) / stat.delta_count; + stat.avg_pack_rows_in_delta = static_cast(stat.total_delta_rows) / stat.total_pack_count_in_delta; + stat.avg_pack_size_in_delta = static_cast(stat.total_delta_size) / stat.total_pack_count_in_delta; + + stat.avg_pack_count_in_stable = static_cast(stat.total_pack_count_in_stable) / stat.stable_count; + stat.avg_pack_rows_in_stable = static_cast(stat.total_stable_rows) / stat.total_pack_count_in_stable; + stat.avg_pack_size_in_stable = static_cast(stat.total_stable_size) / stat.total_pack_count_in_stable; + + static const String useless_tracing_id("DeltaMergeStore::getStat"); + { + auto snaps_stat = storage_pool->dataReader()->getSnapshotsStat(); + stat.storage_stable_num_snapshots = snaps_stat.num_snapshots; + stat.storage_stable_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; + stat.storage_stable_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; + stat.storage_stable_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; + PageStorage::SnapshotPtr stable_snapshot = storage_pool->dataReader()->getSnapshot(useless_tracing_id); + if (const auto * concrete_snap = toConcreteSnapshot(stable_snapshot); concrete_snap != nullptr) + { + if (const auto * const version = concrete_snap->version(); version != nullptr) + { + stat.storage_stable_num_pages = version->numPages(); + stat.storage_stable_num_normal_pages = version->numNormalPages(); + stat.storage_stable_max_page_id = version->maxId(); + } + else + { + LOG_ERROR(log, "Can't get any version from current snapshot, type=data"); + } + } + } + { + auto snaps_stat = storage_pool->logReader()->getSnapshotsStat(); + stat.storage_delta_num_snapshots = snaps_stat.num_snapshots; + stat.storage_delta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; + stat.storage_delta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; + stat.storage_delta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; + PageStorage::SnapshotPtr log_snapshot = storage_pool->logReader()->getSnapshot(useless_tracing_id); + if (const auto * concrete_snap = toConcreteSnapshot(log_snapshot); concrete_snap != nullptr) + { + if (const auto * const version = concrete_snap->version(); version != nullptr) + { + stat.storage_delta_num_pages = version->numPages(); + stat.storage_delta_num_normal_pages = version->numNormalPages(); + stat.storage_delta_max_page_id = version->maxId(); + } + else + { + LOG_ERROR(log, "Can't get any version from current snapshot, type=log"); + } + } + } + { + auto snaps_stat = storage_pool->metaReader()->getSnapshotsStat(); + stat.storage_meta_num_snapshots = snaps_stat.num_snapshots; + stat.storage_meta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; + stat.storage_meta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; + stat.storage_meta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; + PageStorage::SnapshotPtr meta_snapshot = storage_pool->metaReader()->getSnapshot(useless_tracing_id); + if (const auto * concrete_snap = toConcreteSnapshot(meta_snapshot); concrete_snap != nullptr) + { + if (const auto * const version = concrete_snap->version(); version != nullptr) + { + stat.storage_meta_num_pages = version->numPages(); + stat.storage_meta_num_normal_pages = version->numNormalPages(); + stat.storage_meta_max_page_id = version->maxId(); + } + else + { + LOG_ERROR(log, "Can't get any version from current snapshot, type=meta"); + } + } + } + + stat.background_tasks_length = background_tasks.length(); + + return stat; +} + +SegmentsStats DeltaMergeStore::getSegmentsStats() +{ + std::shared_lock lock(read_write_mutex); + + SegmentsStats stats; + for (const auto & [handle, segment] : segments) + { + UNUSED(handle); + + SegmentStats stat; + const auto & delta = segment->getDelta(); + const auto & delta_memtable = delta->getMemTableSet(); + const auto & delta_persisted = delta->getPersistedFileSet(); + const auto & stable = segment->getStable(); + + stat.segment_id = segment->segmentId(); + stat.range = segment->getRowKeyRange(); + stat.epoch = segment->segmentEpoch(); + stat.rows = segment->getEstimatedRows(); + stat.size = segment->getEstimatedBytes(); + + stat.delta_rate = static_cast(delta->getRows()) / stat.rows; + stat.delta_memtable_rows = delta_memtable->getRows(); + stat.delta_memtable_size = delta_memtable->getBytes(); + stat.delta_memtable_column_files = delta_memtable->getColumnFileCount(); + stat.delta_memtable_delete_ranges = delta_memtable->getDeletes(); + stat.delta_persisted_page_id = delta_persisted->getId(); + stat.delta_persisted_rows = delta_persisted->getRows(); + stat.delta_persisted_size = delta_persisted->getBytes(); + stat.delta_persisted_column_files = delta_persisted->getColumnFileCount(); + stat.delta_persisted_delete_ranges = delta_persisted->getDeletes(); + stat.delta_cache_size = delta->getTotalCacheBytes(); + stat.delta_index_size = delta->getDeltaIndexBytes(); + + stat.stable_page_id = stable->getId(); + stat.stable_rows = stable->getRows(); + stat.stable_size = stable->getBytes(); + stat.stable_dmfiles = stable->getDMFiles().size(); + if (stat.stable_dmfiles > 0) + stat.stable_dmfiles_id_0 = stable->getDMFiles().front()->fileId(); + stat.stable_dmfiles_rows = stable->getDMFilesRows(); + stat.stable_dmfiles_size = stable->getDMFilesBytes(); + stat.stable_dmfiles_size_on_disk = stable->getDMFilesBytesOnDisk(); + stat.stable_dmfiles_packs = stable->getDMFilesPacks(); + + stats.emplace_back(stat); + } + return stats; +} + + +} // namespace DM +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/Segment.h b/dbms/src/Storages/DeltaMerge/Segment.h index 52830986e96..665842a351b 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.h +++ b/dbms/src/Storages/DeltaMerge/Segment.h @@ -376,6 +376,7 @@ class Segment : private boost::noncopyable PageId segmentId() const { return segment_id; } PageId nextSegmentId() const { return next_segment_id; } + UInt64 segmentEpoch() const { return epoch; }; void check(DMContext & dm_context, const String & when) const; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index b3a0f41e585..c5d48f0da4f 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -2953,7 +2953,7 @@ try store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); num_rows_write_in_total += num_rows_per_write; - auto segment_stats = store->getSegmentStats(); + auto segment_stats = store->getSegmentsStats(); size_t delta_cache_size = 0; for (auto & stat : segment_stats) { diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp index 4406cf06289..77b629228ec 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp @@ -144,7 +144,7 @@ try } auto delta_store = storage->getStore(); size_t total_segment_rows = 0; - auto segment_stats = delta_store->getSegmentStats(); + auto segment_stats = delta_store->getSegmentsStats(); for (auto & stat : segment_stats) { total_segment_rows += stat.rows; @@ -772,11 +772,11 @@ try { write_data(num_rows_write, 1000); num_rows_write += 1000; - if (storage->getStore()->getSegmentStats().size() > 1) + if (storage->getStore()->getSegmentsStats().size() > 1) break; } { - ASSERT_GT(storage->getStore()->getSegmentStats().size(), 1); + ASSERT_GT(storage->getStore()->getSegmentsStats().size(), 1); ASSERT_EQ(read_data(), num_rows_write); } storage->flushCache(ctx); @@ -793,13 +793,13 @@ try // write more data make sure segments more than 1 for (size_t i = 0; i < 100000; i++) { - if (storage->getStore()->getSegmentStats().size() > 1) + if (storage->getStore()->getSegmentsStats().size() > 1) break; write_data(num_rows_write, 1000); num_rows_write += 1000; } { - ASSERT_GT(storage->getStore()->getSegmentStats().size(), 1); + ASSERT_GT(storage->getStore()->getSegmentsStats().size(), 1); ASSERT_EQ(read_data(), num_rows_write); } storage->flushCache(ctx); @@ -819,7 +819,7 @@ try // restore the table and make sure there is just one segment left create_table(); { - ASSERT_EQ(storage->getStore()->getSegmentStats().size(), 1); + ASSERT_EQ(storage->getStore()->getSegmentsStats().size(), 1); ASSERT_LT(read_data(), num_rows_write); } storage->drop(); diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index 1951383c862..43afe2caee5 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -1362,10 +1362,10 @@ BlockInputStreamPtr StorageDeltaMerge::status() auto & name_col = columns[0]; auto & value_col = columns[1]; - DeltaMergeStoreStat stat; + StoreStats stat; if (storeInited()) { - stat = _store->getStat(); + stat = _store->getStoreStats(); } #define INSERT_INT(NAME) \ diff --git a/dbms/src/Storages/System/StorageSystemDTSegments.cpp b/dbms/src/Storages/System/StorageSystemDTSegments.cpp index f84a19a005c..77dd55e0b1f 100644 --- a/dbms/src/Storages/System/StorageSystemDTSegments.cpp +++ b/dbms/src/Storages/System/StorageSystemDTSegments.cpp @@ -42,22 +42,32 @@ StorageSystemDTSegments::StorageSystemDTSegments(const std::string & name_) {"segment_id", std::make_shared()}, {"range", std::make_shared()}, - + {"epoch", std::make_shared()}, {"rows", std::make_shared()}, {"size", std::make_shared()}, - {"delete_ranges", std::make_shared()}, - - {"stable_size_on_disk", std::make_shared()}, - - {"delta_pack_count", std::make_shared()}, - {"stable_pack_count", std::make_shared()}, - - {"avg_delta_pack_rows", std::make_shared()}, - {"avg_stable_pack_rows", std::make_shared()}, {"delta_rate", std::make_shared()}, + {"delta_memtable_rows", std::make_shared()}, + {"delta_memtable_size", std::make_shared()}, + {"delta_memtable_column_files", std::make_shared()}, + {"delta_memtable_delete_ranges", std::make_shared()}, + {"delta_persisted_page_id", std::make_shared()}, + {"delta_persisted_rows", std::make_shared()}, + {"delta_persisted_size", std::make_shared()}, + {"delta_persisted_column_files", std::make_shared()}, + {"delta_persisted_delete_ranges", std::make_shared()}, {"delta_cache_size", std::make_shared()}, {"delta_index_size", std::make_shared()}, + + {"stable_page_id", std::make_shared()}, + {"stable_rows", std::make_shared()}, + {"stable_size", std::make_shared()}, + {"stable_dmfiles", std::make_shared()}, + {"stable_dmfiles_id_0", std::make_shared()}, + {"stable_dmfiles_rows", std::make_shared()}, + {"stable_dmfiles_size", std::make_shared()}, + {"stable_dmfiles_size_on_disk", std::make_shared()}, + {"stable_dmfiles_packs", std::make_shared()}, })); } @@ -93,7 +103,7 @@ BlockInputStreams StorageSystemDTSegments::read(const Names & column_names, auto dm_storage = std::dynamic_pointer_cast(storage); const auto & table_info = dm_storage->getTableInfo(); auto table_id = table_info.id; - auto segment_stats = dm_storage->getStore()->getSegmentStats(); + auto segment_stats = dm_storage->getStore()->getSegmentsStats(); for (auto & stat : segment_stats) { size_t j = 0; @@ -111,22 +121,32 @@ BlockInputStreams StorageSystemDTSegments::read(const Names & column_names, res_columns[j++]->insert(stat.segment_id); res_columns[j++]->insert(stat.range.toString()); + res_columns[j++]->insert(stat.epoch); res_columns[j++]->insert(stat.rows); res_columns[j++]->insert(stat.size); - res_columns[j++]->insert(stat.delete_ranges); - - res_columns[j++]->insert(stat.stable_size_on_disk); - - res_columns[j++]->insert(stat.delta_pack_count); - res_columns[j++]->insert(stat.stable_pack_count); - - res_columns[j++]->insert(stat.avg_delta_pack_rows); - res_columns[j++]->insert(stat.avg_stable_pack_rows); res_columns[j++]->insert(stat.delta_rate); + res_columns[j++]->insert(stat.delta_memtable_rows); + res_columns[j++]->insert(stat.delta_memtable_size); + res_columns[j++]->insert(stat.delta_memtable_column_files); + res_columns[j++]->insert(stat.delta_memtable_delete_ranges); + res_columns[j++]->insert(stat.delta_persisted_page_id); + res_columns[j++]->insert(stat.delta_persisted_rows); + res_columns[j++]->insert(stat.delta_persisted_size); + res_columns[j++]->insert(stat.delta_persisted_column_files); + res_columns[j++]->insert(stat.delta_persisted_delete_ranges); res_columns[j++]->insert(stat.delta_cache_size); - res_columns[j++]->insert(stat.delta_index_size); + + res_columns[j++]->insert(stat.stable_page_id); + res_columns[j++]->insert(stat.stable_rows); + res_columns[j++]->insert(stat.stable_size); + res_columns[j++]->insert(stat.stable_dmfiles); + res_columns[j++]->insert(stat.stable_dmfiles_id_0); + res_columns[j++]->insert(stat.stable_dmfiles_rows); + res_columns[j++]->insert(stat.stable_dmfiles_size); + res_columns[j++]->insert(stat.stable_dmfiles_size_on_disk); + res_columns[j++]->insert(stat.stable_dmfiles_packs); } } } diff --git a/dbms/src/Storages/System/StorageSystemDTTables.cpp b/dbms/src/Storages/System/StorageSystemDTTables.cpp index b700cfb5324..3e1c37db593 100644 --- a/dbms/src/Storages/System/StorageSystemDTTables.cpp +++ b/dbms/src/Storages/System/StorageSystemDTTables.cpp @@ -145,7 +145,7 @@ BlockInputStreams StorageSystemDTTables::read( auto dm_storage = std::dynamic_pointer_cast(storage); const auto & table_info = dm_storage->getTableInfo(); auto table_id = table_info.id; - auto stat = dm_storage->getStore()->getStat(); + auto stat = dm_storage->getStore()->getStoreStats(); size_t j = 0; res_columns[j++]->insert(database_name);