Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

storage: add more store statistics #6038

Merged
merged 9 commits into from
Oct 5, 2022
+318 −256
Merged

storage: add more store statistics #6038

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6a305d3
Always use readable pathname in DMFileReader (#5958) (#5983)
ti-chi-bot Sep 22, 2022
a05f9e7
Disable folly memcpy as default memcpy (#5996) (#6001)
ti-chi-bot Sep 22, 2022
418fde3
storage: GC segments contained by the pack (#6010)
breezewish Sep 22, 2022
61555a4
Update segment stats
breezewish Sep 24, 2022
b38803c
Merge remote-tracking branch 'origin/master' into wenxuan/segment_info
breezewish Sep 24, 2022
32d61fb
Merge branch 'master' into wenxuan/segment_info
breezewish Sep 27, 2022
24b7923
Merge branch 'master' into wenxuan/segment_info
breezewish Oct 1, 2022
b8f3bad
Address comments
breezewish Oct 5, 2022
14253ad
Merge branch 'master' into wenxuan/segment_info
ti-chi-bot Oct 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dbms/src/Interpreters/AsynchronousMetrics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ void AsynchronousMetrics::update()
{
if (auto store = dt_storage->getStoreIfInited(); store)
{
auto stat = store->getStat();
auto stat = store->getStoreStats();
calculateMax(max_dt_stable_oldest_snapshot_lifetime, stat.storage_stable_oldest_snapshot_lifetime);
calculateMax(max_dt_delta_oldest_snapshot_lifetime, stat.storage_delta_oldest_snapshot_lifetime);
calculateMax(max_dt_meta_oldest_snapshot_lifetime, stat.storage_meta_oldest_snapshot_lifetime);
Expand Down
200 changes: 0 additions & 200 deletions dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1492,206 +1492,6 @@ void DeltaMergeStore::restoreStableFiles()
}
}

static inline DB::PS::V2::PageEntriesVersionSetWithDelta::Snapshot *
toConcreteSnapshot(const DB::PageStorage::SnapshotPtr & ptr)
{
return dynamic_cast<DB::PS::V2::PageEntriesVersionSetWithDelta::Snapshot *>(ptr.get());
}

DeltaMergeStoreStat DeltaMergeStore::getStat()
{
std::shared_lock lock(read_write_mutex);

DeltaMergeStoreStat stat;

if (shutdown_called.load(std::memory_order_relaxed))
return stat;

stat.segment_count = segments.size();

Int64 total_placed_rows = 0;
Int64 total_delta_cache_rows = 0;
Float64 total_delta_cache_size = 0;
Int64 total_delta_valid_cache_rows = 0;
for (const auto & [handle, segment] : segments)
{
(void)handle;
const auto & delta = segment->getDelta();
const auto & stable = segment->getStable();

total_placed_rows += delta->getPlacedDeltaRows();

if (delta->getColumnFileCount())
{
stat.total_rows += delta->getRows();
stat.total_size += delta->getBytes();

stat.total_delete_ranges += delta->getDeletes();

stat.delta_count += 1;
stat.total_pack_count_in_delta += delta->getColumnFileCount();

stat.total_delta_rows += delta->getRows();
stat.total_delta_size += delta->getBytes();

stat.delta_index_size += delta->getDeltaIndexBytes();

total_delta_cache_rows += delta->getTotalCacheRows();
total_delta_cache_size += delta->getTotalCacheBytes();
total_delta_valid_cache_rows += delta->getValidCacheRows();
}

if (stable->getDMFilesPacks())
{
stat.total_rows += stable->getRows();
stat.total_size += stable->getBytes();

stat.stable_count += 1;
stat.total_pack_count_in_stable += stable->getDMFilesPacks();

stat.total_stable_rows += stable->getRows();
stat.total_stable_size += stable->getBytes();
stat.total_stable_size_on_disk += stable->getDMFilesBytesOnDisk();
}
}

stat.delta_rate_rows = static_cast<Float64>(stat.total_delta_rows) / stat.total_rows;
stat.delta_rate_segments = static_cast<Float64>(stat.delta_count) / stat.segment_count;

stat.delta_placed_rate = static_cast<Float64>(total_placed_rows) / stat.total_delta_rows;
stat.delta_cache_size = total_delta_cache_size;
stat.delta_cache_rate = static_cast<Float64>(total_delta_valid_cache_rows) / stat.total_delta_rows;
stat.delta_cache_wasted_rate = static_cast<Float64>(total_delta_cache_rows - total_delta_valid_cache_rows) / total_delta_valid_cache_rows;

stat.avg_segment_rows = static_cast<Float64>(stat.total_rows) / stat.segment_count;
stat.avg_segment_size = static_cast<Float64>(stat.total_size) / stat.segment_count;

stat.avg_delta_rows = static_cast<Float64>(stat.total_delta_rows) / stat.delta_count;
stat.avg_delta_size = static_cast<Float64>(stat.total_delta_size) / stat.delta_count;
stat.avg_delta_delete_ranges = static_cast<Float64>(stat.total_delete_ranges) / stat.delta_count;

stat.avg_stable_rows = static_cast<Float64>(stat.total_stable_rows) / stat.stable_count;
stat.avg_stable_size = static_cast<Float64>(stat.total_stable_size) / stat.stable_count;

stat.avg_pack_count_in_delta = static_cast<Float64>(stat.total_pack_count_in_delta) / stat.delta_count;
stat.avg_pack_rows_in_delta = static_cast<Float64>(stat.total_delta_rows) / stat.total_pack_count_in_delta;
stat.avg_pack_size_in_delta = static_cast<Float64>(stat.total_delta_size) / stat.total_pack_count_in_delta;

stat.avg_pack_count_in_stable = static_cast<Float64>(stat.total_pack_count_in_stable) / stat.stable_count;
stat.avg_pack_rows_in_stable = static_cast<Float64>(stat.total_stable_rows) / stat.total_pack_count_in_stable;
stat.avg_pack_size_in_stable = static_cast<Float64>(stat.total_stable_size) / stat.total_pack_count_in_stable;

static const String useless_tracing_id("DeltaMergeStore::getStat");
{
auto snaps_stat = storage_pool->dataReader()->getSnapshotsStat();
stat.storage_stable_num_snapshots = snaps_stat.num_snapshots;
stat.storage_stable_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds;
stat.storage_stable_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id;
stat.storage_stable_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id;
PageStorage::SnapshotPtr stable_snapshot = storage_pool->dataReader()->getSnapshot(useless_tracing_id);
const auto * concrete_snap = toConcreteSnapshot(stable_snapshot);
if (concrete_snap)
{
if (const auto * const version = concrete_snap->version(); version != nullptr)
{
stat.storage_stable_num_pages = version->numPages();
stat.storage_stable_num_normal_pages = version->numNormalPages();
stat.storage_stable_max_page_id = version->maxId();
}
else
{
LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=data");
}
}
}
{
auto snaps_stat = storage_pool->logReader()->getSnapshotsStat();
stat.storage_delta_num_snapshots = snaps_stat.num_snapshots;
stat.storage_delta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds;
stat.storage_delta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id;
stat.storage_delta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id;
PageStorage::SnapshotPtr log_snapshot = storage_pool->logReader()->getSnapshot(useless_tracing_id);
const auto * concrete_snap = toConcreteSnapshot(log_snapshot);
if (concrete_snap)
{
if (const auto * const version = concrete_snap->version(); version != nullptr)
{
stat.storage_delta_num_pages = version->numPages();
stat.storage_delta_num_normal_pages = version->numNormalPages();
stat.storage_delta_max_page_id = version->maxId();
}
else
{
LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=log");
}
}
}
{
auto snaps_stat = storage_pool->metaReader()->getSnapshotsStat();
stat.storage_meta_num_snapshots = snaps_stat.num_snapshots;
stat.storage_meta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds;
stat.storage_meta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id;
stat.storage_meta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id;
PageStorage::SnapshotPtr meta_snapshot = storage_pool->metaReader()->getSnapshot(useless_tracing_id);
const auto * concrete_snap = toConcreteSnapshot(meta_snapshot);
if (concrete_snap)
{
if (const auto * const version = concrete_snap->version(); version != nullptr)
{
stat.storage_meta_num_pages = version->numPages();
stat.storage_meta_num_normal_pages = version->numNormalPages();
stat.storage_meta_max_page_id = version->maxId();
}
else
{
LOG_FMT_ERROR(log, "Can't get any version from current snapshot, type=meta");
}
}
}

stat.background_tasks_length = background_tasks.length();

return stat;
}

SegmentStats DeltaMergeStore::getSegmentStats()
{
std::shared_lock lock(read_write_mutex);

SegmentStats stats;
for (const auto & [handle, segment] : segments)
{
(void)handle;

SegmentStat stat;
const auto & delta = segment->getDelta();
const auto & stable = segment->getStable();

stat.segment_id = segment->segmentId();
stat.range = segment->getRowKeyRange();

stat.rows = segment->getEstimatedRows();
stat.size = delta->getBytes() + stable->getBytes();
stat.delete_ranges = delta->getDeletes();

stat.stable_size_on_disk = stable->getDMFilesBytesOnDisk();

stat.delta_pack_count = delta->getColumnFileCount();
stat.stable_pack_count = stable->getDMFilesPacks();

stat.avg_delta_pack_rows = static_cast<Float64>(delta->getRows()) / stat.delta_pack_count;
stat.avg_stable_pack_rows = static_cast<Float64>(stable->getRows()) / stat.stable_pack_count;

stat.delta_rate = static_cast<Float64>(delta->getRows()) / stat.rows;
stat.delta_cache_size = delta->getTotalCacheBytes();

stat.delta_index_size = delta->getDeltaIndexBytes();

stats.push_back(stat);
}
return stats;
}

SegmentReadTasks DeltaMergeStore::getReadTasksByRanges(
DMContext & dm_context,
const RowKeyRanges & sorted_ranges,
Expand Down
53 changes: 29 additions & 24 deletions dbms/src/Storages/DeltaMerge/DeltaMergeStore.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,31 +52,40 @@ struct ExternalDTFileInfo;

inline static const PageId DELTA_MERGE_FIRST_SEGMENT_ID = 1;

struct SegmentStat
struct SegmentStats
{
UInt64 segment_id = 0;
RowKeyRange range;

UInt64 epoch = 0;
UInt64 rows = 0;
UInt64 size = 0;
UInt64 delete_ranges = 0;

UInt64 stable_size_on_disk = 0;

UInt64 delta_pack_count = 0;
UInt64 stable_pack_count = 0;

Float64 avg_delta_pack_rows = 0;
Float64 avg_stable_pack_rows = 0;

Float64 delta_rate = 0;
UInt64 delta_memtable_rows = 0;
UInt64 delta_memtable_size = 0;
UInt64 delta_memtable_column_files = 0;
UInt64 delta_memtable_delete_ranges = 0;
UInt64 delta_persisted_page_id = 0;
UInt64 delta_persisted_rows = 0;
UInt64 delta_persisted_size = 0;
UInt64 delta_persisted_column_files = 0;
UInt64 delta_persisted_delete_ranges = 0;
UInt64 delta_cache_size = 0;

UInt64 delta_index_size = 0;

UInt64 stable_page_id = 0;
UInt64 stable_rows = 0;
UInt64 stable_size = 0;
UInt64 stable_dmfiles = 0;
UInt64 stable_dmfiles_id_0 = 0;
UInt64 stable_dmfiles_rows = 0;
UInt64 stable_dmfiles_size = 0;
UInt64 stable_dmfiles_size_on_disk = 0;
UInt64 stable_dmfiles_packs = 0;
};
using SegmentStats = std::vector<SegmentStat>;
using SegmentsStats = std::vector<SegmentStats>;

struct DeltaMergeStoreStat
struct StoreStats
{
UInt64 segment_count = 0;

Expand Down Expand Up @@ -385,16 +394,12 @@ class DeltaMergeStore : private boost::noncopyable
SortDescription getPrimarySortDescription() const;

void check(const Context & db_context);
DeltaMergeStoreStat getStat();
SegmentStats getSegmentStats();
bool isCommonHandle() const
{
return is_common_handle;
}
size_t getRowKeyColumnSize() const
{
return rowkey_column_size;
}

StoreStats getStoreStats();
SegmentsStats getSegmentsStats();

bool isCommonHandle() const { return is_common_handle; }
size_t getRowKeyColumnSize() const { return rowkey_column_size; }

public:
/// Methods mainly used by region split.
Expand Down
Loading