NEED

zhiqiang-hhhh · Oct 16, 2024 · 2745f12 · 2745f12
1 parent 07ba5eb
commit 2745f12
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 15 deletions.
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -59,7 +59,6 @@ Status OlapScanLocalState::_init_profile() {
     _block_fetch_timer = ADD_TIMER(_scanner_profile, "BlockFetchTime");
     _delete_bitmap_get_agg_timer = ADD_TIMER(_scanner_profile, "DeleteBitmapGetAggTime");
     _sync_rowset_timer = ADD_TIMER(_scanner_profile, "SyncRowsetTime");
-    _raw_rows_counter = ADD_COUNTER(_segment_profile, "RawRowsRead", TUnit::UNIT);
     _block_convert_timer = ADD_TIMER(_scanner_profile, "BlockConvertTime");
     _block_init_timer = ADD_TIMER(_segment_profile, "BlockInitTime");
     _block_init_seek_timer = ADD_TIMER(_segment_profile, "BlockInitSeekTime");

diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h
@@ -110,7 +110,6 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
     RuntimeProfile::Counter* _read_compressed_counter = nullptr;
     RuntimeProfile::Counter* _decompressor_timer = nullptr;
     RuntimeProfile::Counter* _read_uncompressed_counter = nullptr;
-    RuntimeProfile::Counter* _raw_rows_counter = nullptr;
 
     RuntimeProfile::Counter* _rows_vec_cond_filtered_counter = nullptr;
     RuntimeProfile::Counter* _rows_short_circuit_cond_filtered_counter = nullptr;

diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp
@@ -1071,6 +1071,13 @@ Status ScanLocalState<Derived>::_init_profile() {
 
     _peak_running_scanner =
             _scanner_profile->AddHighWaterMarkCounter("PeakRunningScanner", TUnit::UNIT);
+
+    // Rows read from storage.
+    // Include the rows read from doris page cache.
+    _scan_rows = ADD_COUNTER(_runtime_profile, "ScanRows", TUnit::UNIT);
+    // Size of data that read from storage.
+    // Does not include rows that are cached by doris page cache.
+    _scan_bytes = ADD_COUNTER(_runtime_profile, "ScanBytes", TUnit::BYTES);
     return Status::OK();
 }
 

diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h
@@ -128,6 +128,9 @@ class ScanLocalStateBase : public PipelineXLocalState<>, public RuntimeFilterCon
     RuntimeProfile::Counter* _num_scanners = nullptr;
 
     RuntimeProfile::Counter* _wait_for_rf_timer = nullptr;
+
+    RuntimeProfile::Counter* _scan_rows = nullptr;
+    RuntimeProfile::Counter* _scan_bytes = nullptr;
 };
 
 template <typename LocalStateType>

diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -540,14 +540,15 @@ Status NewOlapScanner::close(RuntimeState* state) {
 void NewOlapScanner::_update_realtime_counters() {
     pipeline::OlapScanLocalState* local_state =
             static_cast<pipeline::OlapScanLocalState*>(_local_state);
-    auto& stats = _tablet_reader->stats();
+    const OlapReaderStatistics& stats = _tablet_reader->stats();
     COUNTER_UPDATE(local_state->_read_compressed_counter, stats.compressed_bytes_read);
-    _compressed_bytes_read += stats.compressed_bytes_read;
+    COUNTER_UPDATE(local_state->_scan_bytes, stats.compressed_bytes_read);
+    _scan_bytes += stats.compressed_bytes_read;
     _tablet_reader->mutable_stats()->compressed_bytes_read = 0;
 
-    COUNTER_UPDATE(local_state->_raw_rows_counter, stats.raw_rows_read);
+    COUNTER_UPDATE(local_state->_scan_rows, stats.raw_rows_read);
+    _scan_rows += stats.raw_rows_read;
     // if raw_rows_read is reset, scanNode will scan all table rows which may cause BE crash
-    _raw_rows_read += stats.raw_rows_read;
     _tablet_reader->mutable_stats()->raw_rows_read = 0;
 }
 
@@ -564,16 +565,17 @@ void NewOlapScanner::_collect_profile_before_close() {
 #define INCR_COUNTER(Parent)                                                                      \
     COUNTER_UPDATE(Parent->_io_timer, stats.io_ns);                                               \
     COUNTER_UPDATE(Parent->_read_compressed_counter, stats.compressed_bytes_read);                \
-    _compressed_bytes_read += stats.compressed_bytes_read;                                        \
+    COUNTER_UPDATE(Parent->_scan_bytes, stats.compressed_bytes_read);                             \
+    _scan_bytes += stats.compressed_bytes_read;                                                   \
     COUNTER_UPDATE(Parent->_decompressor_timer, stats.decompress_ns);                             \
     COUNTER_UPDATE(Parent->_read_uncompressed_counter, stats.uncompressed_bytes_read);            \
     COUNTER_UPDATE(Parent->_block_load_timer, stats.block_load_ns);                               \
     COUNTER_UPDATE(Parent->_block_load_counter, stats.blocks_load);                               \
     COUNTER_UPDATE(Parent->_block_fetch_timer, stats.block_fetch_ns);                             \
     COUNTER_UPDATE(Parent->_delete_bitmap_get_agg_timer, stats.delete_bitmap_get_agg_ns);         \
     COUNTER_UPDATE(Parent->_block_convert_timer, stats.block_convert_ns);                         \
-    COUNTER_UPDATE(Parent->_raw_rows_counter, stats.raw_rows_read);                               \
-    _raw_rows_read += _tablet_reader->mutable_stats()->raw_rows_read;                             \
+    COUNTER_UPDATE(Parent->_scan_rows, stats.raw_rows_read);                                      \
+    _scan_rows += _tablet_reader->mutable_stats()->raw_rows_read;                                 \
     COUNTER_UPDATE(Parent->_vec_cond_timer, stats.vec_cond_ns);                                   \
     COUNTER_UPDATE(Parent->_short_cond_timer, stats.short_cond_ns);                               \
     COUNTER_UPDATE(Parent->_expr_filter_timer, stats.expr_filter_ns);                             \
@@ -663,11 +665,11 @@ void NewOlapScanner::_collect_profile_before_close() {
 #undef INCR_COUNTER
 #endif
     // Update metrics
-    DorisMetrics::instance()->query_scan_bytes->increment(_compressed_bytes_read);
-    DorisMetrics::instance()->query_scan_rows->increment(_raw_rows_read);
+    DorisMetrics::instance()->query_scan_bytes->increment(_scan_bytes);
+    DorisMetrics::instance()->query_scan_rows->increment(_scan_rows);
     auto& tablet = _tablet_reader_params.tablet;
-    tablet->query_scan_bytes->increment(_compressed_bytes_read);
-    tablet->query_scan_rows->increment(_raw_rows_read);
+    tablet->query_scan_bytes->increment(_scan_bytes);
+    tablet->query_scan_rows->increment(_scan_rows);
     tablet->query_scan_count->increment(1);
     if (_query_statistics) {
         _query_statistics->add_scan_bytes_from_local_storage(

diff --git a/be/src/vec/exec/scan/new_olap_scanner.h b/be/src/vec/exec/scan/new_olap_scanner.h
@@ -101,8 +101,8 @@ class NewOlapScanner : public VScanner {
     std::unordered_set<uint32_t> _tablet_columns_convert_to_null_set;
 
     // ========= profiles ==========
-    int64_t _compressed_bytes_read = 0;
-    int64_t _raw_rows_read = 0;
+    int64_t _scan_bytes = 0;
+    int64_t _scan_rows = 0;
     bool _profile_updated = false;
 };
 } // namespace vectorized