diff --git a/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToBlockInputStream.cpp index 39084174cce..f5534998684 100644 --- a/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToBlockInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToBlockInputStream.cpp @@ -51,7 +51,9 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( // , prehandle_task(prehandle_task_) , opts(std::move(opts_)) { - log = Logger::get(opts.log_prefix); + const size_t split_id + = soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT; + log = Logger::get(opts.log_prefix, fmt::format("region_id={} split_id={}", region->id(), split_id)); // We have to initialize sst readers at an earlier stage, // due to prehandle snapshot of single region feature in raftstore v2. @@ -62,9 +64,8 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( // auto make_inner_func = [&](const TiFlashRaftProxyHelper * proxy_helper, SSTView snap, SSTReader::RegionRangeFilter range, - size_t split_id, - size_t region_id) { - return std::make_unique(proxy_helper, snap, range, split_id, region_id); + const LoggerPtr & log_) { + return std::make_unique(proxy_helper, snap, range, log_); }; for (UInt64 i = 0; i < snaps.len; ++i) { @@ -92,9 +93,7 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( // make_inner_func, ssts_default, log, - region->getRange(), - soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT, - region->id()); + region->getRange()); } if (!ssts_write.empty()) { @@ -104,9 +103,7 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( // make_inner_func, ssts_write, log, - region->getRange(), - soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT, - region->id()); + region->getRange()); } if (!ssts_lock.empty()) { @@ -116,9 +113,7 @@ SSTFilesToBlockInputStream::SSTFilesToBlockInputStream( // make_inner_func, ssts_lock, log, - region->getRange(), - soft_limit.has_value() ? soft_limit.value().split_id : DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT, - region->id()); + region->getRange()); } LOG_INFO( log, diff --git a/dbms/src/Storages/KVStore/FFI/SSTReader.cpp b/dbms/src/Storages/KVStore/FFI/SSTReader.cpp index 36c0bc15905..2578dc0507f 100644 --- a/dbms/src/Storages/KVStore/FFI/SSTReader.cpp +++ b/dbms/src/Storages/KVStore/FFI/SSTReader.cpp @@ -109,17 +109,14 @@ MonoSSTReader::MonoSSTReader( const TiFlashRaftProxyHelper * proxy_helper_, SSTView view, RegionRangeFilter range_, - size_t split_id_, - size_t region_id_) + const LoggerPtr & log_) : proxy_helper(proxy_helper_) , inner(proxy_helper->sst_reader_interfaces.fn_get_sst_reader(view, proxy_helper->proxy_ptr)) , type(view.type) , range(range_) , tail_checked(false) - , split_id(split_id_) - , region_id(region_id_) + , log(log_) { - log = &Poco::Logger::get("MonoSSTReader"); kind = proxy_helper->sst_reader_interfaces.fn_kind(inner, view.type); if (kind == SSTFormatKind::KIND_TABLET) { @@ -128,11 +125,9 @@ MonoSSTReader::MonoSSTReader( // 'z' will be added in proxy. LOG_INFO( log, - "Seek cf {} to {}, split_id={} region_id={}", + "Seek cf {} to {}", magic_enum::enum_name(type), - Redact::keyToDebugString(start.data(), start.size()), - split_id, - region_id); + Redact::keyToDebugString(start.data(), start.size())); if (!start.empty()) { proxy_helper->sst_reader_interfaces @@ -141,11 +136,6 @@ MonoSSTReader::MonoSSTReader( } } -size_t MonoSSTReader::getSplitId() const -{ - return split_id; -} - MonoSSTReader::~MonoSSTReader() { proxy_helper->sst_reader_interfaces.fn_gc(inner, type); diff --git a/dbms/src/Storages/KVStore/FFI/SSTReader.h b/dbms/src/Storages/KVStore/FFI/SSTReader.h index 46195e216d1..6bdd1015631 100644 --- a/dbms/src/Storages/KVStore/FFI/SSTReader.h +++ b/dbms/src/Storages/KVStore/FFI/SSTReader.h @@ -35,7 +35,6 @@ class SSTReader virtual void seek(BaseBuffView && view) const = 0; virtual void seekToFirst() const = 0; virtual void seekToLast() const = 0; - virtual size_t getSplitId() const = 0; virtual ~SSTReader() = default; }; @@ -54,15 +53,13 @@ class MonoSSTReader : public SSTReader void seek(BaseBuffView && view) const override; void seekToFirst() const override; void seekToLast() const override; - size_t getSplitId() const override; DISALLOW_COPY_AND_MOVE(MonoSSTReader); MonoSSTReader( const TiFlashRaftProxyHelper * proxy_helper_, SSTView view, RegionRangeFilter range_, - size_t split_id_, - size_t region_id_); + const LoggerPtr & log_); ~MonoSSTReader() override; private: @@ -72,9 +69,7 @@ class MonoSSTReader : public SSTReader RegionRangeFilter range; SSTFormatKind kind; mutable bool tail_checked; - size_t split_id; - size_t region_id; - Poco::Logger * log; + LoggerPtr log; }; /// MultiSSTReader helps when there are multiple sst files in a column family. @@ -89,7 +84,7 @@ class MultiSSTReader : public SSTReader { public: using Initer - = std::function(const TiFlashRaftProxyHelper *, E, RegionRangeFilter, size_t, size_t)>; + = std::function(const TiFlashRaftProxyHelper *, E, RegionRangeFilter, const LoggerPtr &)>; DISALLOW_COPY_AND_MOVE(MultiSSTReader); @@ -148,7 +143,6 @@ class MultiSSTReader : public SSTReader } return mono->seekToLast(); } - size_t getSplitId() const override { return split_id; } // Switch to next mono reader if current SST is drained, // and we have a next sst file to read. @@ -164,14 +158,12 @@ class MultiSSTReader : public SSTReader // and it will be dropped as MultiSSTReader is dropped. LOG_INFO( log, - "Open sst file {}, range={} sst_idx={} sst_tot={} split_id={} region_id={}", + "Open sst file {}, range={} sst_idx={} sst_tot={}", buffToStrView(args[sst_idx].path), range->toDebugString(), sst_idx, - args.size(), - split_id, - region_id); - mono = initer(proxy_helper, args[sst_idx], range, split_id, region_id); + args.size()); + mono = initer(proxy_helper, args[sst_idx], range, log); } } @@ -181,9 +173,7 @@ class MultiSSTReader : public SSTReader Initer initer_, std::vector args_, LoggerPtr log_, - RegionRangeFilter range_, - size_t split_id_, - size_t region_id_) + RegionRangeFilter range_) : log(log_) , proxy_helper(proxy_helper_) , type(type_) @@ -191,19 +181,15 @@ class MultiSSTReader : public SSTReader , args(args_) , sst_idx(0) , range(range_) - , split_id(split_id_) - , region_id(region_id_) { assert(args.size() > 0); LOG_INFO( log, - "Open sst file first {}, range={} sst_tot={} split_id={} region_id={}", + "Open sst file first {}, range={} sst_tot={}", buffToStrView(args[sst_idx].path), range->toDebugString(), - args.size(), - split_id, - region_id); - mono = initer(proxy_helper, args[sst_idx], range, split_id, region_id); + args.size()); + mono = initer(proxy_helper, args[sst_idx], range, log); } ~MultiSSTReader() override @@ -222,8 +208,6 @@ class MultiSSTReader : public SSTReader std::vector args; size_t sst_idx; RegionRangeFilter range; - const size_t split_id; - const size_t region_id; }; } // namespace DB diff --git a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp index 191c6a76e05..64a6aa631fd 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp +++ b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp @@ -138,20 +138,17 @@ static inline std::tuple executeTransform const auto & opts = prehandle_ctx.opts; auto & tmt = prehandle_ctx.tmt; auto & trace = prehandle_ctx.trace; + const auto split_id = sst_stream->getSplitId(); + const String limit_tag = sst_stream->getSoftLimit() ? sst_stream->getSoftLimit()->toDebugString() : ""; + const auto region_id = new_region->id(); - auto region_id = new_region->id(); - auto split_id = sst_stream->getSplitId(); CurrentMetrics::add(CurrentMetrics::RaftNumPrehandlingSubTasks); SCOPE_EXIT({ trace.releaseSubtaskResources(region_id, split_id); CurrentMetrics::sub(CurrentMetrics::RaftNumPrehandlingSubTasks); }); Stopwatch sw; - LOG_INFO( - log, - "Add prehandle task split_id={} limit={}", - split_id, - sst_stream->getSoftLimit().has_value() ? sst_stream->getSoftLimit()->toDebugString() : ""); + LOG_INFO(log, "Add prehandle task split_id={} limit={}", split_id, limit_tag); std::shared_ptr> stream; // If any schema changes is detected during decoding SSTs to DTFiles, we need to cancel and recreate DTFiles with // the latest schema. Or we will get trouble in `BoundedSSTFilesToBlockInputStream`. @@ -520,6 +517,7 @@ std::tuple executeParallelTransform( split_key_count, new_region->id(), snaps.len); + Stopwatch watch; // Make sure the queue is bigger than `split_key_count`, otherwise `addTask` may fail. auto async_tasks = SingleSnapshotAsyncTasks(split_key_count, split_key_count, split_key_count + 5); diff --git a/dbms/src/Storages/KVStore/tests/region_kvstore_test.h b/dbms/src/Storages/KVStore/tests/region_kvstore_test.h index e53356a6cd3..248c74ff0f5 100644 --- a/dbms/src/Storages/KVStore/tests/region_kvstore_test.h +++ b/dbms/src/Storages/KVStore/tests/region_kvstore_test.h @@ -59,10 +59,9 @@ inline void validateSSTGeneration( auto make_inner_func = [](const TiFlashRaftProxyHelper * proxy_helper, SSTView snap, SSTReader::RegionRangeFilter range, - size_t split_id, - size_t region_id) -> std::unique_ptr { + const LoggerPtr & log_) -> std::unique_ptr { auto parsed_kind = MockSSTGenerator::parseSSTViewKind(buffToStrView(snap.path)); - auto reader = std::make_unique(proxy_helper, snap, range, split_id, region_id); + auto reader = std::make_unique(proxy_helper, snap, range, log_); assert(reader->sstFormatKind() == parsed_kind); return reader; }; @@ -73,8 +72,7 @@ inline void validateSSTGeneration( ssts, Logger::get(), kvr1->getRange(), - DM::SSTScanSoftLimit::HEAD_OR_ONLY_SPLIT, - region_id}; + }; size_t counter = 0; while (reader.remained())