Skip to content

Commit

Permalink
Simplify region_id, split_id logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@JaySon-Huang
JaySon-Huang committed Jul 8, 2024
1 parent 44df65f commit f83d2cb
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 65 deletions.
21 changes: 8 additions & 13 deletions dbms/src/Storages/DeltaMerge/Decode/SSTFilesToBlockInputStream.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( //
, prehandle_task(prehandle_task_)
, opts(std::move(opts_))
{
log = Logger::get(opts.log_prefix);
const size_t split_id
= soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT;
log = Logger::get(opts.log_prefix, fmt::format("region_id={} split_id={}", region->id(), split_id));

// We have to initialize sst readers at an earlier stage,
// due to prehandle snapshot of single region feature in raftstore v2.
Expand All @@ -62,9 +64,8 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( //
auto make_inner_func = [&](const TiFlashRaftProxyHelper * proxy_helper,
SSTView snap,
SSTReader::RegionRangeFilter range,
size_t split_id,
size_t region_id) {
return std::make_unique<MonoSSTReader>(proxy_helper, snap, range, split_id, region_id);
const LoggerPtr & log_) {
return std::make_unique<MonoSSTReader>(proxy_helper, snap, range, log_);
};
for (UInt64 i = 0; i < snaps.len; ++i)
{
Expand Down Expand Up @@ -92,9 +93,7 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( //
make_inner_func,
ssts_default,
log,
region->getRange(),
soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT,
region->id());
region->getRange());
}
if (!ssts_write.empty())
{
Expand All @@ -104,9 +103,7 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( //
make_inner_func,
ssts_write,
log,
region->getRange(),
soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT,
region->id());
region->getRange());
}
if (!ssts_lock.empty())
{
Expand All @@ -116,9 +113,7 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( //
make_inner_func,
ssts_lock,
log,
region->getRange(),
soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT,
region->id());
region->getRange());
}
LOG_INFO(
log,
Expand Down
18 changes: 4 additions & 14 deletions dbms/src/Storages/KVStore/FFI/SSTReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,17 +109,14 @@ MonoSSTReader::MonoSSTReader(
const TiFlashRaftProxyHelper * proxy_helper_,
SSTView view,
RegionRangeFilter range_,
size_t split_id_,
size_t region_id_)
const LoggerPtr & log_)
: proxy_helper(proxy_helper_)
, inner(proxy_helper->sst_reader_interfaces.fn_get_sst_reader(view, proxy_helper->proxy_ptr))
, type(view.type)
, range(range_)
, tail_checked(false)
, split_id(split_id_)
, region_id(region_id_)
, log(log_)
{
log = &Poco::Logger::get("MonoSSTReader");
kind = proxy_helper->sst_reader_interfaces.fn_kind(inner, view.type);
if (kind == SSTFormatKind::KIND_TABLET)
{
Expand All @@ -128,11 +125,9 @@ MonoSSTReader::MonoSSTReader(
// 'z' will be added in proxy.
LOG_INFO(
log,
"Seek cf {} to {}, split_id={} region_id={}",
"Seek cf {} to {}",
magic_enum::enum_name(type),
Redact::keyToDebugString(start.data(), start.size()),
split_id,
region_id);
Redact::keyToDebugString(start.data(), start.size()));
if (!start.empty())
{
proxy_helper->sst_reader_interfaces
Expand All @@ -141,11 +136,6 @@ MonoSSTReader::MonoSSTReader(
}
}

size_t MonoSSTReader::getSplitId() const
{
return split_id;
}

MonoSSTReader::~MonoSSTReader()
{
proxy_helper->sst_reader_interfaces.fn_gc(inner, type);
Expand Down
36 changes: 10 additions & 26 deletions dbms/src/Storages/KVStore/FFI/SSTReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ class SSTReader
virtual void seek(BaseBuffView && view) const = 0;
virtual void seekToFirst() const = 0;
virtual void seekToLast() const = 0;
virtual size_t getSplitId() const = 0;

virtual ~SSTReader() = default;
};
Expand All @@ -54,15 +53,13 @@ class MonoSSTReader : public SSTReader
void seek(BaseBuffView && view) const override;
void seekToFirst() const override;
void seekToLast() const override;
size_t getSplitId() const override;

DISALLOW_COPY_AND_MOVE(MonoSSTReader);
MonoSSTReader(
const TiFlashRaftProxyHelper * proxy_helper_,
SSTView view,
RegionRangeFilter range_,
size_t split_id_,
size_t region_id_);
const LoggerPtr & log_);
~MonoSSTReader() override;

private:
Expand All @@ -72,9 +69,7 @@ class MonoSSTReader : public SSTReader
RegionRangeFilter range;
SSTFormatKind kind;
mutable bool tail_checked;
size_t split_id;
size_t region_id;
Poco::Logger * log;
LoggerPtr log;
};

/// MultiSSTReader helps when there are multiple sst files in a column family.
Expand All @@ -89,7 +84,7 @@ class MultiSSTReader : public SSTReader
{
public:
using Initer
= std::function<std::unique_ptr<R>(const TiFlashRaftProxyHelper *, E, RegionRangeFilter, size_t, size_t)>;
= std::function<std::unique_ptr<R>(const TiFlashRaftProxyHelper *, E, RegionRangeFilter, const LoggerPtr &)>;

DISALLOW_COPY_AND_MOVE(MultiSSTReader);

Expand Down Expand Up @@ -148,7 +143,6 @@ class MultiSSTReader : public SSTReader
}
return mono->seekToLast();
}
size_t getSplitId() const override { return split_id; }

// Switch to next mono reader if current SST is drained,
// and we have a next sst file to read.
Expand All @@ -164,14 +158,12 @@ class MultiSSTReader : public SSTReader
// and it will be dropped as MultiSSTReader is dropped.
LOG_INFO(
log,
"Open sst file {}, range={} sst_idx={} sst_tot={} split_id={} region_id={}",
"Open sst file {}, range={} sst_idx={} sst_tot={}",
buffToStrView(args[sst_idx].path),
range->toDebugString(),
sst_idx,
args.size(),
split_id,
region_id);
mono = initer(proxy_helper, args[sst_idx], range, split_id, region_id);
args.size());
mono = initer(proxy_helper, args[sst_idx], range, log);
}
}

Expand All @@ -181,29 +173,23 @@ class MultiSSTReader : public SSTReader
Initer initer_,
std::vector<E> args_,
LoggerPtr log_,
RegionRangeFilter range_,
size_t split_id_,
size_t region_id_)
RegionRangeFilter range_)
: log(log_)
, proxy_helper(proxy_helper_)
, type(type_)
, initer(initer_)
, args(args_)
, sst_idx(0)
, range(range_)
, split_id(split_id_)
, region_id(region_id_)
{
assert(args.size() > 0);
LOG_INFO(
log,
"Open sst file first {}, range={} sst_tot={} split_id={} region_id={}",
"Open sst file first {}, range={} sst_tot={}",
buffToStrView(args[sst_idx].path),
range->toDebugString(),
args.size(),
split_id,
region_id);
mono = initer(proxy_helper, args[sst_idx], range, split_id, region_id);
args.size());
mono = initer(proxy_helper, args[sst_idx], range, log);
}

~MultiSSTReader() override
Expand All @@ -222,8 +208,6 @@ class MultiSSTReader : public SSTReader
std::vector<E> args;
size_t sst_idx;
RegionRangeFilter range;
const size_t split_id;
const size_t region_id;
};

} // namespace DB
12 changes: 5 additions & 7 deletions dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,20 +138,17 @@ static inline std::tuple<ReadFromStreamResult, PrehandleResult> executeTransform
const auto & opts = prehandle_ctx.opts;
auto & tmt = prehandle_ctx.tmt;
auto & trace = prehandle_ctx.trace;
const auto split_id = sst_stream->getSplitId();
const String limit_tag = sst_stream->getSoftLimit() ? sst_stream->getSoftLimit()->toDebugString() : "";
const auto region_id = new_region->id();

auto region_id = new_region->id();
auto split_id = sst_stream->getSplitId();
CurrentMetrics::add(CurrentMetrics::RaftNumPrehandlingSubTasks);
SCOPE_EXIT({
trace.releaseSubtaskResources(region_id, split_id);
CurrentMetrics::sub(CurrentMetrics::RaftNumPrehandlingSubTasks);
});
Stopwatch sw;
LOG_INFO(
log,
"Add prehandle task split_id={} limit={}",
split_id,
sst_stream->getSoftLimit().has_value() ? sst_stream->getSoftLimit()->toDebugString() : "");
LOG_INFO(log, "Add prehandle task split_id={} limit={}", split_id, limit_tag);
std::shared_ptr<DM::SSTFilesToDTFilesOutputStream<DM::BoundedSSTFilesToBlockInputStreamPtr>> stream;
// If any schema changes is detected during decoding SSTs to DTFiles, we need to cancel and recreate DTFiles with
// the latest schema. Or we will get trouble in `BoundedSSTFilesToBlockInputStream`.
Expand Down Expand Up @@ -520,6 +517,7 @@ std::tuple<ReadFromStreamResult, PrehandleResult> executeParallelTransform(
split_key_count,
new_region->id(),
snaps.len);

Stopwatch watch;
// Make sure the queue is bigger than `split_key_count`, otherwise `addTask` may fail.
auto async_tasks = SingleSnapshotAsyncTasks(split_key_count, split_key_count, split_key_count + 5);
Expand Down
8 changes: 3 additions & 5 deletions dbms/src/Storages/KVStore/tests/region_kvstore_test.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,9 @@ inline void validateSSTGeneration(
auto make_inner_func = [](const TiFlashRaftProxyHelper * proxy_helper,
SSTView snap,
SSTReader::RegionRangeFilter range,
size_t split_id,
size_t region_id) -> std::unique_ptr<MonoSSTReader> {
const LoggerPtr & log_) -> std::unique_ptr<MonoSSTReader> {
auto parsed_kind = MockSSTGenerator::parseSSTViewKind(buffToStrView(snap.path));
auto reader = std::make_unique<MonoSSTReader>(proxy_helper, snap, range, split_id, region_id);
auto reader = std::make_unique<MonoSSTReader>(proxy_helper, snap, range, log_);
assert(reader->sstFormatKind() == parsed_kind);
return reader;
};
Expand All @@ -73,8 +72,7 @@ inline void validateSSTGeneration(
ssts,
Logger::get(),
kvr1->getRange(),
DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT,
region_id};
};

size_t counter = 0;
while (reader.remained())
Expand Down

0 comments on commit f83d2cb

Please sign in to comment.