Skip to content

Commit

Permalink
Add flow by different types
Browse files Browse the repository at this point in the history 
Signed-off-by: JaySon-Huang <[email protected]>
  • Loading branch information
@JaySon-Huang
JaySon-Huang committed Apr 7, 2023
1 parent c8b0be9 commit e831153
Show file tree
Hide file tree
Showing 6 changed files with 118 additions and 9 deletions.
19 changes: 14 additions & 5 deletions dbms/src/Common/TiFlashMetrics.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,14 @@ namespace DB
F(type_num_files, {"type", "num_files"})) \
M(tiflash_storage_checkpoint_flow, "The bytes flow cause by remote checkpoint", Counter, \
F(type_incremental, {"type", "incremental"}), F(type_compaction, {"type", "compaction"})) \
M(tiflash_storage_checkpoint_keys_by_types, "The keys flow cause by remote checkpoint", Counter, \
F(type_raftengine, {"type", "raftengine"}), F(type_kvengine, {"type", "kvengine"}), F(type_kvstore, {"type", "kvstore"}), \
F(type_data, {"type", "data"}), F(type_log, {"type", "log"}), F(type_meta, {"type", "kvstore"}), \
F(type_unknown, {"type", "unknown"})) \
M(tiflash_storage_checkpoint_flow_by_types, "The bytes flow cause by remote checkpoint", Counter, \
F(type_raftengine, {"type", "raftengine"}), F(type_kvengine, {"type", "kvengine"}), F(type_kvstore, {"type", "kvstore"}), \
F(type_data, {"type", "data"}), F(type_log, {"type", "log"}), F(type_meta, {"type", "kvstore"}), \
F(type_unknown, {"type", "unknown"})) \
M(tiflash_storage_s3_request_seconds, "S3 request duration in seconds", Histogram, \
F(type_put_object, {{"type", "put_object"}}, ExpBuckets{0.001, 2, 20}), \
F(type_copy_object, {{"type", "copy_object"}}, ExpBuckets{0.001, 2, 20}), \
Expand All @@ -345,11 +353,12 @@ namespace DB
F(type_clean_manifests, {{"type", "clean_manifests"}}, ExpBuckets{0.5, 2, 20}), \
F(type_scan_then_clean_data_files, {{"type", "scan_then_clean_data_files"}}, ExpBuckets{0.5, 2, 20}), \
F(type_clean_one_lock, {{"type", "clean_one_lock"}}, ExpBuckets{0.5, 2, 20})) \
M(tiflash_storage_checkpoint_seconds, "PageStorage checkpoint elapsed time", Histogram, \
F(type_dump_checkpoint_snapshot, {{"type", "dump_checkpoint_snapshot"}}, ExpBuckets{0.001, 2, 20}), \
F(type_dump_checkpoint_data, {{"type", "dump_checkpoint_data"}}, ExpBuckets{0.001, 2, 20}), \
F(type_upload_checkpoint, {{"type", "upload_checkpoint"}}, ExpBuckets{0.001, 2, 20}), \
F(type_copy_checkpoint_info, {{"type", "copy_checkpoint_info"}}, ExpBuckets{0.001, 2, 20}))
M(tiflash_storage_checkpoint_seconds, "PageStorage checkpoint elapsed time", \
Histogram, /* these command usually cost several seconds, increase the start bucket to 50ms */ \
F(type_dump_checkpoint_snapshot, {{"type", "dump_checkpoint_snapshot"}}, ExpBuckets{0.05, 2, 20}), \
F(type_dump_checkpoint_data, {{"type", "dump_checkpoint_data"}}, ExpBuckets{0.05, 2, 20}), \
F(type_upload_checkpoint, {{"type", "upload_checkpoint"}}, ExpBuckets{0.05, 2, 20}), \
F(type_copy_checkpoint_info, {{"type", "copy_checkpoint_info"}}, ExpBuckets{0.05, 2, 20}))

// clang-format on

Expand Down
78 changes: 78 additions & 0 deletions dbms/src/Storages/Page/V3/CheckpointFile/CPDumpStat.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright 2023 PingCAP, Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Common/TiFlashMetrics.h>
#include <Storages/Page/PageConstants.h>
#include <Storages/Page/V3/CheckpointFile/CPDumpStat.h>
#include <fmt/core.h>

namespace DB::PS::V3
{

void SetMetrics(const CPDataDumpStats & stats)
{
for (size_t i = 0; i < static_cast<size_t>(DB::StorageType::_MAX_STORAGE_TYPE_); ++i)
{
auto type = static_cast<DB::StorageType>(i);
switch (type)
{
case DB::StorageType::Unknown:
{
GET_METRIC(tiflash_storage_checkpoint_keys_by_types, type_unknown).Increment(stats.num_keys[i]);
GET_METRIC(tiflash_storage_checkpoint_flow_by_types, type_unknown).Increment(stats.num_bytes[i]);
break;
}
case DB::StorageType::RaftEngine:
{
GET_METRIC(tiflash_storage_checkpoint_keys_by_types, type_raftengine).Increment(stats.num_keys[i]);
GET_METRIC(tiflash_storage_checkpoint_flow_by_types, type_raftengine).Increment(stats.num_bytes[i]);
break;
}
case DB::StorageType::KVEngine:
{
GET_METRIC(tiflash_storage_checkpoint_keys_by_types, type_kvengine).Increment(stats.num_keys[i]);
GET_METRIC(tiflash_storage_checkpoint_flow_by_types, type_kvengine).Increment(stats.num_bytes[i]);
break;
}
case DB::StorageType::KVStore:
{
GET_METRIC(tiflash_storage_checkpoint_keys_by_types, type_kvstore).Increment(stats.num_keys[i]);
GET_METRIC(tiflash_storage_checkpoint_flow_by_types, type_kvstore).Increment(stats.num_bytes[i]);
break;
}
case DB::StorageType::Data:
{
GET_METRIC(tiflash_storage_checkpoint_keys_by_types, type_data).Increment(stats.num_keys[i]);
GET_METRIC(tiflash_storage_checkpoint_flow_by_types, type_data).Increment(stats.num_bytes[i]);
break;
}
case DB::StorageType::Log:
{
GET_METRIC(tiflash_storage_checkpoint_keys_by_types, type_log).Increment(stats.num_keys[i]);
GET_METRIC(tiflash_storage_checkpoint_flow_by_types, type_log).Increment(stats.num_bytes[i]);
break;
}
case DB::StorageType::Meta:
{
GET_METRIC(tiflash_storage_checkpoint_keys_by_types, type_meta).Increment(stats.num_keys[i]);
GET_METRIC(tiflash_storage_checkpoint_flow_by_types, type_meta).Increment(stats.num_bytes[i]);
break;
}
default:
__builtin_unreachable();
}
}
}

} // namespace DB::PS::V3
12 changes: 11 additions & 1 deletion dbms/src/Storages/Page/V3/CheckpointFile/CPDumpStat.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,16 @@ struct CPDataDumpStats
size_t num_pages_incremental = 0;
// Number of ExternalPages
size_t num_ext_pages = 0;
// Number of RefPages
size_t num_ref_pages = 0;
// Number of delete records
size_t num_delete_records = 0;
// Number of other records other than Pages/ExternalPages
size_t num_other_records = 0;
};

void SetMetrics(const CPDataDumpStats & stats);

} // namespace DB::PS::V3

template <>
Expand All @@ -62,14 +68,18 @@ struct fmt::formatter<DB::PS::V3::CPDataDumpStats>
ctx.out(),
"CPDataDumpStats{{"
"incremental_data_bytes={} compact_data_bytes={}"
" n_records{{total={} pages_unchanged={} pages_compact={} pages_incremental={} ext_pages={} other={}}}",
" n_records{{total={}"
" pages_unchanged={} pages_compact={} pages_incremental={} ext_pages={} ref_pages={}"
" delete={} other={}}}",
value.incremental_data_bytes,
value.compact_data_bytes,
value.num_records,
value.num_pages_unchanged,
value.num_pages_compact,
value.num_pages_incremental,
value.num_ext_pages,
value.num_ref_pages,
value.num_delete_records,
value.num_other_records);
it = format_to(it, " types[");
for (size_t i = 0; i < static_cast<size_t>(DB::StorageType::_MAX_STORAGE_TYPE_); ++i)
Expand Down
14 changes: 13 additions & 1 deletion dbms/src/Storages/Page/V3/CheckpointFile/CPFilesWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

#include <Storages/Page/V3/CheckpointFile/CPDumpStat.h>
#include <Storages/Page/V3/CheckpointFile/CPFilesWriter.h>
#include <Storages/Page/V3/PageEntriesEdit.h>
#include <Storages/Page/V3/PageEntryCheckpointInfo.h>
#include <Storages/Page/V3/Universal/UniversalPageIdFormatImpl.h>
#include <fmt/core.h>
Expand Down Expand Up @@ -112,7 +113,18 @@ CPDataDumpStats CPFilesWriter::writeEditsAndApplyCheckpointInfo(

if (rec_edit.type != EditRecordType::VAR_ENTRY)
{
write_down_stats.num_other_records += 1;
if (rec_edit.type == EditRecordType::VAR_REF)
{
write_down_stats.num_ref_pages += 1;
}
else if (rec_edit.type == EditRecordType::VAR_DELETE)
{
write_down_stats.num_delete_records += 1;
}
else
{
write_down_stats.num_other_records += 1;
}
continue;
}

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/S3/S3GCManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ void S3GCManager::removeDataFileIfDelmarkExpired(
const String & delmark_key,
const Aws::Utils::DateTime & timepoint,
const Aws::Utils::DateTime & delmark_mtime,
const LoggerPtr & sub_logger)
const LoggerPtr & sub_logger) const
{
// delmark exist
bool expired = false;
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/S3/S3GCManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ class S3GCManager
const String & delmark_key,
const Aws::Utils::DateTime & timepoint,
const Aws::Utils::DateTime & delmark_mtime,
const LoggerPtr & sub_logger);
const LoggerPtr & sub_logger) const;

void lifecycleMarkDataFileDeleted(const String & datafile_key, const LoggerPtr & sub_logger);
void physicalRemoveDataFile(const String & datafile_key, const LoggerPtr & sub_logger) const;
Expand Down

0 comments on commit e831153

Please sign in to comment.