Skip to content

Commit

Permalink
storage: add more store statistics (#6038)
Browse files Browse the repository at this point in the history 
ref #5975
  • Loading branch information
@breezewish
breezewish authored Oct 5, 2022
1 parent d9d8f29 commit 5e6f620
Show file tree
Hide file tree
Showing 10 changed files with 318 additions and 256 deletions.
2 changes: 1 addition & 1 deletion dbms/src/Interpreters/AsynchronousMetrics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ void AsynchronousMetrics::update()
{
if (auto store = dt_storage->getStoreIfInited(); store)
{
auto stat = store->getStat();
auto stat = store->getStoreStats();
calculateMax(max_dt_stable_oldest_snapshot_lifetime, stat.storage_stable_oldest_snapshot_lifetime);
calculateMax(max_dt_delta_oldest_snapshot_lifetime, stat.storage_delta_oldest_snapshot_lifetime);
calculateMax(max_dt_meta_oldest_snapshot_lifetime, stat.storage_meta_oldest_snapshot_lifetime);
Expand Down
200 changes: 0 additions & 200 deletions dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1492,206 +1492,6 @@ void DeltaMergeStore::restoreStableFiles()
}
}

static inline DB::PS::V2::PageEntriesVersionSetWithDelta::Snapshot *
toConcreteSnapshot(const DB::PageStorage::SnapshotPtr & ptr)
{
return dynamic_cast<DB::PS::V2::PageEntriesVersionSetWithDelta::Snapshot *>(ptr.get());
}

DeltaMergeStoreStat DeltaMergeStore::getStat()
{
std::shared_lock lock(read_write_mutex);

DeltaMergeStoreStat stat;

if (shutdown_called.load(std::memory_order_relaxed))
return stat;

stat.segment_count = segments.size();

Int64 total_placed_rows = 0;
Int64 total_delta_cache_rows = 0;
Float64 total_delta_cache_size = 0;
Int64 total_delta_valid_cache_rows = 0;
for (const auto & [handle, segment] : segments)
{
(void)handle;
const auto & delta = segment->getDelta();
const auto & stable = segment->getStable();

total_placed_rows += delta->getPlacedDeltaRows();

if (delta->getColumnFileCount())
{
stat.total_rows += delta->getRows();
stat.total_size += delta->getBytes();

stat.total_delete_ranges += delta->getDeletes();

stat.delta_count += 1;
stat.total_pack_count_in_delta += delta->getColumnFileCount();

stat.total_delta_rows += delta->getRows();
stat.total_delta_size += delta->getBytes();

stat.delta_index_size += delta->getDeltaIndexBytes();

total_delta_cache_rows += delta->getTotalCacheRows();
total_delta_cache_size += delta->getTotalCacheBytes();
total_delta_valid_cache_rows += delta->getValidCacheRows();
}

if (stable->getDMFilesPacks())
{
stat.total_rows += stable->getRows();
stat.total_size += stable->getBytes();

stat.stable_count += 1;
stat.total_pack_count_in_stable += stable->getDMFilesPacks();

stat.total_stable_rows += stable->getRows();
stat.total_stable_size += stable->getBytes();
stat.total_stable_size_on_disk += stable->getDMFilesBytesOnDisk();
}
}

stat.delta_rate_rows = static_cast<Float64>(stat.total_delta_rows) / stat.total_rows;
stat.delta_rate_segments = static_cast<Float64>(stat.delta_count) / stat.segment_count;

stat.delta_placed_rate = static_cast<Float64>(total_placed_rows) / stat.total_delta_rows;
stat.delta_cache_size = total_delta_cache_size;
stat.delta_cache_rate = static_cast<Float64>(total_delta_valid_cache_rows) / stat.total_delta_rows;
stat.delta_cache_wasted_rate = static_cast<Float64>(total_delta_cache_rows - total_delta_valid_cache_rows) / total_delta_valid_cache_rows;

stat.avg_segment_rows = static_cast<Float64>(stat.total_rows) / stat.segment_count;
stat.avg_segment_size = static_cast<Float64>(stat.total_size) / stat.segment_count;

stat.avg_delta_rows = static_cast<Float64>(stat.total_delta_rows) / stat.delta_count;
stat.avg_delta_size = static_cast<Float64>(stat.total_delta_size) / stat.delta_count;
stat.avg_delta_delete_ranges = static_cast<Float64>(stat.total_delete_ranges) / stat.delta_count;

stat.avg_stable_rows = static_cast<Float64>(stat.total_stable_rows) / stat.stable_count;
stat.avg_stable_size = static_cast<Float64>(stat.total_stable_size) / stat.stable_count;

stat.avg_pack_count_in_delta = static_cast<Float64>(stat.total_pack_count_in_delta) / stat.delta_count;
stat.avg_pack_rows_in_delta = static_cast<Float64>(stat.total_delta_rows) / stat.total_pack_count_in_delta;
stat.avg_pack_size_in_delta = static_cast<Float64>(stat.total_delta_size) / stat.total_pack_count_in_delta;

stat.avg_pack_count_in_stable = static_cast<Float64>(stat.total_pack_count_in_stable) / stat.stable_count;
stat.avg_pack_rows_in_stable = static_cast<Float64>(stat.total_stable_rows) / stat.total_pack_count_in_stable;
stat.avg_pack_size_in_stable = static_cast<Float64>(stat.total_stable_size) / stat.total_pack_count_in_stable;

static const String useless_tracing_id("DeltaMergeStore::getStat");
{
auto snaps_stat = storage_pool->dataReader()->getSnapshotsStat();
stat.storage_stable_num_snapshots = snaps_stat.num_snapshots;
stat.storage_stable_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds;
stat.storage_stable_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id;
stat.storage_stable_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id;
PageStorage::SnapshotPtr stable_snapshot = storage_pool->dataReader()->getSnapshot(useless_tracing_id);
const auto * concrete_snap = toConcreteSnapshot(stable_snapshot);
if (concrete_snap)
{
if (const auto * const version = concrete_snap->version(); version != nullptr)
{
stat.storage_stable_num_pages = version->numPages();
stat.storage_stable_num_normal_pages = version->numNormalPages();
stat.storage_stable_max_page_id = version->maxId();
}
else
{
LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=data");
}
}
}
{
auto snaps_stat = storage_pool->logReader()->getSnapshotsStat();
stat.storage_delta_num_snapshots = snaps_stat.num_snapshots;
stat.storage_delta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds;
stat.storage_delta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id;
stat.storage_delta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id;
PageStorage::SnapshotPtr log_snapshot = storage_pool->logReader()->getSnapshot(useless_tracing_id);
const auto * concrete_snap = toConcreteSnapshot(log_snapshot);
if (concrete_snap)
{
if (const auto * const version = concrete_snap->version(); version != nullptr)
{
stat.storage_delta_num_pages = version->numPages();
stat.storage_delta_num_normal_pages = version->numNormalPages();
stat.storage_delta_max_page_id = version->maxId();
}
else
{
LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=log");
}
}
}
{
auto snaps_stat = storage_pool->metaReader()->getSnapshotsStat();
stat.storage_meta_num_snapshots = snaps_stat.num_snapshots;
stat.storage_meta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds;
stat.storage_meta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id;
stat.storage_meta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id;
PageStorage::SnapshotPtr meta_snapshot = storage_pool->metaReader()->getSnapshot(useless_tracing_id);
const auto * concrete_snap = toConcreteSnapshot(meta_snapshot);
if (concrete_snap)
{
if (const auto * const version = concrete_snap->version(); version != nullptr)
{
stat.storage_meta_num_pages = version->numPages();
stat.storage_meta_num_normal_pages = version->numNormalPages();
stat.storage_meta_max_page_id = version->maxId();
}
else
{
LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=meta");
}
}
}

stat.background_tasks_length = background_tasks.length();

return stat;
}

SegmentStats DeltaMergeStore::getSegmentStats()
{
std::shared_lock lock(read_write_mutex);

SegmentStats stats;
for (const auto & [handle, segment] : segments)
{
(void)handle;

SegmentStat stat;
const auto & delta = segment->getDelta();
const auto & stable = segment->getStable();

stat.segment_id = segment->segmentId();
stat.range = segment->getRowKeyRange();

stat.rows = segment->getEstimatedRows();
stat.size = delta->getBytes() + stable->getBytes();
stat.delete_ranges = delta->getDeletes();

stat.stable_size_on_disk = stable->getDMFilesBytesOnDisk();

stat.delta_pack_count = delta->getColumnFileCount();
stat.stable_pack_count = stable->getDMFilesPacks();

stat.avg_delta_pack_rows = static_cast<Float64>(delta->getRows()) / stat.delta_pack_count;
stat.avg_stable_pack_rows = static_cast<Float64>(stable->getRows()) / stat.stable_pack_count;

stat.delta_rate = static_cast<Float64>(delta->getRows()) / stat.rows;
stat.delta_cache_size = delta->getTotalCacheBytes();

stat.delta_index_size = delta->getDeltaIndexBytes();

stats.push_back(stat);
}
return stats;
}

SegmentReadTasks DeltaMergeStore::getReadTasksByRanges(
DMContext & dm_context,
const RowKeyRanges & sorted_ranges,
Expand Down
53 changes: 29 additions & 24 deletions dbms/src/Storages/DeltaMerge/DeltaMergeStore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,31 +52,40 @@ struct ExternalDTFileInfo;

inline static const PageId DELTA_MERGE_FIRST_SEGMENT_ID = 1;

struct SegmentStat
struct SegmentStats
{
UInt64 segment_id = 0;
RowKeyRange range;

UInt64 epoch = 0;
UInt64 rows = 0;
UInt64 size = 0;
UInt64 delete_ranges = 0;

UInt64 stable_size_on_disk = 0;

UInt64 delta_pack_count = 0;
UInt64 stable_pack_count = 0;

Float64 avg_delta_pack_rows = 0;
Float64 avg_stable_pack_rows = 0;

Float64 delta_rate = 0;
UInt64 delta_memtable_rows = 0;
UInt64 delta_memtable_size = 0;
UInt64 delta_memtable_column_files = 0;
UInt64 delta_memtable_delete_ranges = 0;
UInt64 delta_persisted_page_id = 0;
UInt64 delta_persisted_rows = 0;
UInt64 delta_persisted_size = 0;
UInt64 delta_persisted_column_files = 0;
UInt64 delta_persisted_delete_ranges = 0;
UInt64 delta_cache_size = 0;

UInt64 delta_index_size = 0;

UInt64 stable_page_id = 0;
UInt64 stable_rows = 0;
UInt64 stable_size = 0;
UInt64 stable_dmfiles = 0;
UInt64 stable_dmfiles_id_0 = 0;
UInt64 stable_dmfiles_rows = 0;
UInt64 stable_dmfiles_size = 0;
UInt64 stable_dmfiles_size_on_disk = 0;
UInt64 stable_dmfiles_packs = 0;
};
using SegmentStats = std::vector<SegmentStat>;
using SegmentsStats = std::vector<SegmentStats>;

struct DeltaMergeStoreStat
struct StoreStats
{
UInt64 segment_count = 0;

Expand Down Expand Up @@ -385,16 +394,12 @@ class DeltaMergeStore : private boost::noncopyable
SortDescription getPrimarySortDescription() const;

void check(const Context & db_context);
DeltaMergeStoreStat getStat();
SegmentStats getSegmentStats();
bool isCommonHandle() const
{
return is_common_handle;
}
size_t getRowKeyColumnSize() const
{
return rowkey_column_size;
}

StoreStats getStoreStats();
SegmentsStats getSegmentsStats();

bool isCommonHandle() const { return is_common_handle; }
size_t getRowKeyColumnSize() const { return rowkey_column_size; }

public:
/// Methods mainly used by region split.
Expand Down
Loading

0 comments on commit 5e6f620

Please sign in to comment.