From 2da4538dd5f7fa1a4e77d62959626a5da780e51d Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 7 Apr 2023 13:54:58 +0800 Subject: [PATCH 1/2] Fix data race in join when `Join::cancel()` is called (#7228) ref pingcap/tiflash#6528 --- .../CreatingSetsBlockInputStream.cpp | 2 +- .../HashJoinProbeBlockInputStream.cpp | 46 ++++++++------- .../HashJoinProbeBlockInputStream.h | 1 + .../DataStreams/NonJoinedBlockInputStream.cpp | 10 +++- dbms/src/Interpreters/Join.cpp | 57 ++++++++++++------- dbms/src/Interpreters/Join.h | 18 +++--- 6 files changed, 80 insertions(+), 54 deletions(-) diff --git a/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp b/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp index 95f43695350..b0d27768729 100644 --- a/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -109,7 +109,7 @@ void CreatingSetsBlockInputStream::createAll() for (auto & elem : subqueries_for_sets) { if (elem.second.join) - elem.second.join->setInitActiveBuildConcurrency(); + elem.second.join->setInitActiveBuildThreads(); } } Stopwatch watch; diff --git a/dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp index 65143969bd9..0c82b6dd671 100644 --- a/dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp +++ b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp @@ -37,26 +37,23 @@ HashJoinProbeBlockInputStream::HashJoinProbeBlockInputStream( RUNTIME_CHECK_MSG(join != nullptr, "join ptr should not be null."); RUNTIME_CHECK_MSG(join->getProbeConcurrency() > 0, "Join probe concurrency must be greater than 0"); + auto input_header = input->getHeader(); + assert(input_header.rows() == 0); if (need_output_non_joined_data) - non_joined_stream = join->createStreamWithNonJoinedRows(input->getHeader(), current_non_joined_stream_index, join->getProbeConcurrency(), max_block_size); + non_joined_stream = join->createStreamWithNonJoinedRows(input_header, current_non_joined_stream_index, join->getProbeConcurrency(), max_block_size); + ProbeProcessInfo header_probe_process_info(0); + header_probe_process_info.resetBlock(std::move(input_header)); + header = original_join->joinBlock(header_probe_process_info, true); } Block HashJoinProbeBlockInputStream::getHeader() const { - Block res = children.back()->getHeader(); - assert(res.rows() == 0); - ProbeProcessInfo header_probe_process_info(0); - header_probe_process_info.resetBlock(std::move(res)); - /// use original_join here so we don't need add lock - return original_join->joinBlock(header_probe_process_info); + return header; } void HashJoinProbeBlockInputStream::cancel(bool kill) { IProfilingBlockInputStream::cancel(kill); - /// When the probe stream quits probe by cancelling instead of normal finish, the Join operator might still produce meaningless blocks - /// and expects these meaningless blocks won't be used to produce meaningful result. - JoinPtr current_join; RestoreInfo restore_info; { @@ -66,18 +63,23 @@ void HashJoinProbeBlockInputStream::cancel(bool kill) restore_info.build_stream = restore_build_stream; restore_info.probe_stream = restore_probe_stream; } - /// Cancel join just wake up all the threads waiting in Join::waitUntilAllBuildFinished/Join::waitUntilAllProbeFinished, - /// the ongoing join process will not be interrupted - /// There is a little bit hack here because cancel will be called in two cases: - /// 1. the query is cancelled by the caller or meet error: in this case, wake up all waiting threads is safe - /// 2. the query is executed normally, and one of the data stream has read an empty block, the the data stream and all its children - /// will call `cancel(false)`, in this case, there is two sub-cases - /// a. the data stream read an empty block because of EOF, then it means there must be no threads waiting in Join, so cancel the join is safe - /// b. the data stream read an empty block because of early exit of some executor(like limit), in this case, just wake the waiting - /// threads is not 100% safe because if the probe thread is wake up when build is not finished yet, it may produce wrong results, for now - /// it is safe because when any of the data stream read empty block because of early exit, the execution framework ensures that no further - /// data will be used. - current_join->cancel(); + /// Join::wakeUpAllWaitingThreads wakes up all the threads waiting in Join::waitUntilAllBuildFinished/waitUntilAllProbeFinished, + /// and once this function is called, all the subsequent call of Join::waitUntilAllBuildFinished/waitUntilAllProbeFinished will + /// skip waiting directly. + /// HashJoinProbeBlockInputStream::cancel will be called in two cases: + /// 1. the query is cancelled by the caller or meet error: in this case, wake up all waiting threads is safe, because no data + /// will be used data anymore + /// 2. the query is executed normally, and one of the data stream has read an empty block, the the data stream and all its + /// children will call `cancel(false)`, in this case, there is two sub-cases + /// a. the data stream read an empty block because of EOF, then it means there must be no threads waiting in Join, so wake + /// up all waiting threads is safe because actually there is no threads to be waken up + /// b. the data stream read an empty block because of early exit of some executor(like limit), in this case, waking up the + /// waiting threads is not 100% safe because if the probe thread is waken up when build is not finished yet, it may get + /// wrong result. Currently, the execution framework ensures that when any of the data stream read empty block because + /// of early exit, no further data will be used, and in order to make sure no wrong result is generated + /// - for threads reading joined data: will return empty block if build is not finished yet + /// - for threads reading non joined data: will return empty block if build or probe is not finished yet + current_join->wakeUpAllWaitingThreads(); if (restore_info.non_joined_stream != nullptr) { auto * p_stream = dynamic_cast(restore_info.non_joined_stream.get()); diff --git a/dbms/src/DataStreams/HashJoinProbeBlockInputStream.h b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.h index f643de3a1ea..d0d21858879 100644 --- a/dbms/src/DataStreams/HashJoinProbeBlockInputStream.h +++ b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.h @@ -139,6 +139,7 @@ class HashJoinProbeBlockInputStream : public IProfilingBlockInputStream size_t non_joined_rows = 0; std::list parents; std::list> probe_partition_blocks; + Block header; }; } // namespace DB diff --git a/dbms/src/DataStreams/NonJoinedBlockInputStream.cpp b/dbms/src/DataStreams/NonJoinedBlockInputStream.cpp index 11f2d541ff1..dac8f92c5ba 100644 --- a/dbms/src/DataStreams/NonJoinedBlockInputStream.cpp +++ b/dbms/src/DataStreams/NonJoinedBlockInputStream.cpp @@ -131,10 +131,16 @@ Block NonJoinedBlockInputStream::readImpl() /// If build concurrency is less than non join concurrency, /// just return empty block for extra non joined block input stream read if (unlikely(index >= parent.getBuildConcurrency())) - return Block(); + return {}; + if unlikely (parent.active_build_threads != 0 || parent.active_probe_threads != 0) + { + /// build/probe is not finished yet, the query must be cancelled, so just return {} + LOG_WARNING(parent.log, "NonJoinedBlock read without non zero active_build_threads/active_probe_threads, return empty block"); + return {}; + } if (!parent.has_build_data_in_memory) /// no build data in memory, the non joined result must be empty - return Block(); + return {}; /// todo read data based on JoinPartition if (add_not_mapped_rows) diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 245696fca5c..0225394d2be 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -194,9 +194,9 @@ Join::Join( , key_names_left(key_names_left_) , key_names_right(key_names_right_) , build_concurrency(0) - , active_build_concurrency(0) + , active_build_threads(0) , probe_concurrency(0) - , active_probe_concurrency(0) + , active_probe_threads(0) , collators(collators_) , non_equal_conditions(non_equal_conditions_) , original_strictness(strictness) @@ -613,7 +613,7 @@ void Join::setBuildConcurrencyAndInitJoinPartition(size_t build_concurrency_) { if (unlikely(build_concurrency > 0)) throw Exception("Logical error: `setBuildConcurrencyAndInitJoinPartition` shouldn't be called more than once", ErrorCodes::LOGICAL_ERROR); - /// do not set active_build_concurrency because in compile stage, `joinBlock` will be called to get generate header, if active_build_concurrency + /// do not set active_build_threads because in compile stage, `joinBlock` will be called to get generate header, if active_build_threads /// is set here, `joinBlock` will hang when used to get header build_concurrency = std::max(1, build_concurrency_); @@ -2728,12 +2728,12 @@ void Join::checkTypesOfKeys(const Block & block_left, const Block & block_right) void Join::finishOneBuild() { std::unique_lock lock(build_probe_mutex); - if (active_build_concurrency == 1) + if (active_build_threads == 1) { FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_mpp_hash_build); } - --active_build_concurrency; - if (active_build_concurrency == 0) + --active_build_threads; + if (active_build_threads == 0) { workAfterBuildFinish(); build_cv.notify_all(); @@ -2807,7 +2807,7 @@ void Join::waitUntilAllBuildFinished() const { std::unique_lock lock(build_probe_mutex); build_cv.wait(lock, [&]() { - return active_build_concurrency == 0 || meet_error || is_canceled; + return active_build_threads == 0 || meet_error || skip_wait; }); if (meet_error) throw Exception(error_message); @@ -2816,12 +2816,12 @@ void Join::waitUntilAllBuildFinished() const void Join::finishOneProbe() { std::unique_lock lock(build_probe_mutex); - if (active_probe_concurrency == 1) + if (active_probe_threads == 1) { FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_mpp_hash_probe); } - --active_probe_concurrency; - if (active_probe_concurrency == 0) + --active_probe_threads; + if (active_probe_threads == 0) { workAfterProbeFinish(); probe_cv.notify_all(); @@ -2832,7 +2832,7 @@ void Join::waitUntilAllProbeFinished() const { std::unique_lock lock(build_probe_mutex); probe_cv.wait(lock, [&]() { - return active_probe_concurrency == 0 || meet_error || is_canceled; + return active_probe_threads == 0 || meet_error || skip_wait; }); if (meet_error) throw Exception(error_message); @@ -2841,21 +2841,38 @@ void Join::waitUntilAllProbeFinished() const void Join::finishOneNonJoin(size_t partition_index) { - while (partition_index < build_concurrency) + if likely (active_build_threads == 0 && active_probe_threads == 0) { - std::unique_lock partition_lock = partitions[partition_index]->lockPartition(); - partitions[partition_index]->releaseBuildPartitionBlocks(partition_lock); - partitions[partition_index]->releaseProbePartitionBlocks(partition_lock); - if (!partitions[partition_index]->isSpill()) + /// only clear hash table if not active build/probe threads + while (partition_index < build_concurrency) { - releaseBuildPartitionHashTable(partition_index, partition_lock); + std::unique_lock partition_lock = partitions[partition_index]->lockPartition(); + partitions[partition_index]->releaseBuildPartitionBlocks(partition_lock); + partitions[partition_index]->releaseProbePartitionBlocks(partition_lock); + if (!partitions[partition_index]->isSpill()) + { + releaseBuildPartitionHashTable(partition_index, partition_lock); + } + partition_index += build_concurrency; } - partition_index += build_concurrency; } } -Block Join::joinBlock(ProbeProcessInfo & probe_process_info) const +Block Join::joinBlock(ProbeProcessInfo & probe_process_info, bool dry_run) const { + if unlikely (dry_run) + { + assert(probe_process_info.block.rows() == 0); + } + else + { + if unlikely (active_build_threads != 0) + { + /// build is not finished yet, the query must be cancelled, so just return {} + LOG_WARNING(log, "JoinBlock without non zero active_build_threads, return empty block"); + return {}; + } + } std::shared_lock lock(rwlock); probe_process_info.updateStartRow(); @@ -3169,7 +3186,7 @@ RestoreInfo Join::getOneRestoreStream(size_t max_block_size_) auto new_max_bytes_before_external_join = static_cast(max_bytes_before_external_join * (static_cast(restore_join_build_concurrency) / build_concurrency)); restore_join = createRestoreJoin(std::max(1, new_max_bytes_before_external_join)); restore_join->initBuild(build_sample_block, restore_join_build_concurrency); - restore_join->setInitActiveBuildConcurrency(); + restore_join->setInitActiveBuildThreads(); restore_join->initProbe(probe_sample_block, restore_join_build_concurrency); for (Int64 i = 0; i < restore_join_build_concurrency; i++) { diff --git a/dbms/src/Interpreters/Join.h b/dbms/src/Interpreters/Join.h index 8c1ae142747..aa44703a707 100644 --- a/dbms/src/Interpreters/Join.h +++ b/dbms/src/Interpreters/Join.h @@ -128,7 +128,7 @@ class Join /** Join data from the map (that was previously built by calls to insertFromBlock) to the block with data from "left" table. * Could be called from different threads in parallel. */ - Block joinBlock(ProbeProcessInfo & probe_process_info) const; + Block joinBlock(ProbeProcessInfo & probe_process_info, bool dry_run = false) const; void checkTypes(const Block & block) const; @@ -177,10 +177,10 @@ class Join const Names & getLeftJoinKeys() const { return key_names_left; } - void setInitActiveBuildConcurrency() + void setInitActiveBuildThreads() { std::unique_lock lock(build_probe_mutex); - active_build_concurrency = getBuildConcurrency(); + active_build_threads = getBuildConcurrency(); } size_t getProbeConcurrency() const @@ -192,13 +192,13 @@ class Join { std::unique_lock lock(build_probe_mutex); probe_concurrency = concurrency; - active_probe_concurrency = probe_concurrency; + active_probe_threads = probe_concurrency; } - void cancel() + void wakeUpAllWaitingThreads() { std::unique_lock lk(build_probe_mutex); - is_canceled = true; + skip_wait = true; probe_cv.notify_all(); build_cv.notify_all(); } @@ -366,13 +366,13 @@ class Join mutable std::condition_variable build_cv; size_t build_concurrency; - size_t active_build_concurrency; + std::atomic active_build_threads; mutable std::condition_variable probe_cv; size_t probe_concurrency; - size_t active_probe_concurrency; + std::atomic active_probe_threads; - bool is_canceled = false; + bool skip_wait = false; bool meet_error = false; String error_message; From b8786b07ee3e4b500b08f94d9956ed9ac5ce252c Mon Sep 17 00:00:00 2001 From: Wenxuan Date: Fri, 7 Apr 2023 15:24:58 +0800 Subject: [PATCH 2/2] Improve metrics for BR import (#7246) close pingcap/tiflash#7245 --- metrics/grafana/tiflash_summary.json | 2122 ++++++++++++++------------ 1 file changed, 1109 insertions(+), 1013 deletions(-) diff --git a/metrics/grafana/tiflash_summary.json b/metrics/grafana/tiflash_summary.json index 6510029a237..4851ebce1b2 100644 --- a/metrics/grafana/tiflash_summary.json +++ b/metrics/grafana/tiflash_summary.json @@ -52,7 +52,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1679634291456, + "iteration": 1680849886690, "links": [], "panels": [ { @@ -513,7 +513,7 @@ "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, @@ -552,9 +552,10 @@ "seriesOverrides": [ { "alias": "/limit/", - "color": "#C4162A", - "fill": 0, - "nullPointMode": "null" + "color": "#F2495C", + "hideTooltip": true, + "legend": false, + "linewidth": 2 } ], "spaceLength": 10, @@ -710,7 +711,7 @@ "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { @@ -748,16 +749,12 @@ "points": false, "renderer": "flot", "seriesOverrides": [ - { - "alias": "total", - "fill": 0, - "lines": false - }, { "alias": "/limit/", - "color": "#C4162A", - "fill": 0, - "nullPointMode": "null" + "color": "#F2495C", + "hideTooltip": true, + "legend": false, + "linewidth": 2 } ], "spaceLength": 10, @@ -831,13 +828,16 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of fsync operations.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", + "description": "", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, @@ -845,7 +845,7 @@ "y": 17 }, "hiddenSeries": false, - "id": 52, + "id": 181, "legend": { "alignAsTable": true, "avg": false, @@ -863,7 +863,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -878,19 +878,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "exemplar": true, + "expr": "sum by (instance) (irate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}[1m]))", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", - "refId": "A" + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "FSync OPS", + "title": "IO Throughput", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -905,8 +910,8 @@ }, "yaxes": [ { - "decimals": null, - "format": "ops", + "decimals": 0, + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -914,46 +919,64 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 139, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of open file descriptors action.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", + "decimals": null, + "description": "Involved when importing data.", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 24 + "y": 25 }, "hiddenSeries": false, - "id": 22, + "id": 141, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -983,29 +1006,24 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Open-{{instance}}", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"sst_importer.*\"}[1m]))", "format": "time_series", + "hide": false, + "instant": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "OpenFail-{{instance}}", - "refId": "B" + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "File Open OPS", + "title": "SST Import Service", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1020,7 +1038,8 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -1047,27 +1066,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of currently opened file descriptors.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", + "decimals": null, + "description": "Involved when importing data.", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 24 + "y": 25 }, "hiddenSeries": false, - "id": 50, + "id": 154, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, - "hideZero": false, "max": true, "min": false, "rightSide": true, @@ -1090,56 +1111,49 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "Limit", + "color": "#F2495C", + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "nullPointMode": "connected" + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ - { - "expr": "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "W-{{instance}}", - "refId": "B" - }, { "exemplar": true, - "expr": "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"apply_low_.*\"}[1m]))", "format": "time_series", "hide": false, + "instant": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "R-{{instance}}", - "refId": "C" + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 }, { "exemplar": true, - "expr": "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"apply_low_.*\"})", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "RW-{{instance}}", - "refId": "D" + "intervalFactor": 2, + "legendFormat": "Limit", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Opened File Count", + "title": "SST Apply", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1154,7 +1168,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -1174,23 +1189,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Server", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 139, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -1198,7 +1197,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, - "description": "Involved when importing data.", + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -1212,10 +1211,10 @@ "h": 7, "w": 12, "x": 0, - "y": 2 + "y": 32 }, "hiddenSeries": false, - "id": 141, + "id": 145, "legend": { "alignAsTable": true, "avg": false, @@ -1247,6 +1246,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -1257,19 +1257,19 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"sst_importer.*\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_task.*\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "{{name}} {{instance}}", "refId": "A", "step": 40 }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"sst_importer.*\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_task.*\"})", "hide": false, "interval": "", "intervalFactor": 2, @@ -1281,7 +1281,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "SST Import Service", + "title": "Region Task", "tooltip": { "msResolution": false, "shared": true, @@ -1327,7 +1327,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, - "description": "Involved when importing data.", + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -1341,10 +1341,10 @@ "h": 7, "w": 12, "x": 12, - "y": 2 + "y": 32 }, "hiddenSeries": false, - "id": 154, + "id": 147, "legend": { "alignAsTable": true, "avg": false, @@ -1376,6 +1376,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -1386,19 +1387,19 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"apply_low_.*\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_worker.*\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "{{name}} {{instance}}", "refId": "A", "step": 40 }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"apply_low_.*\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_worker.*\"})", "hide": false, "interval": "", "intervalFactor": 2, @@ -1410,7 +1411,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "SST Apply", + "title": "Region Worker", "tooltip": { "msResolution": false, "shared": true, @@ -1470,10 +1471,10 @@ "h": 7, "w": 12, "x": 0, - "y": 9 + "y": 39 }, "hiddenSeries": false, - "id": 145, + "id": 155, "legend": { "alignAsTable": true, "avg": false, @@ -1505,6 +1506,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -1515,7 +1517,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_task.*\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"raftstore_.*\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -1527,7 +1529,7 @@ }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_task.*\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"raftstore_.*\"})", "hide": false, "interval": "", "intervalFactor": 2, @@ -1539,7 +1541,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Region Task", + "title": "Raft Store", "tooltip": { "msResolution": false, "shared": true, @@ -1599,10 +1601,10 @@ "h": 7, "w": 12, "x": 12, - "y": 9 + "y": 39 }, "hiddenSeries": false, - "id": 147, + "id": 153, "legend": { "alignAsTable": true, "avg": false, @@ -1634,6 +1636,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -1644,7 +1647,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_worker.*\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"snap_sender.*\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -1656,7 +1659,7 @@ }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"region_worker.*\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"snap_sender.*\"})", "hide": false, "interval": "", "intervalFactor": 2, @@ -1668,7 +1671,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Region Worker", + "title": "Snapshot Sender", "tooltip": { "msResolution": false, "shared": true, @@ -1728,10 +1731,10 @@ "h": 7, "w": 12, "x": 0, - "y": 16 + "y": 46 }, "hiddenSeries": false, - "id": 155, + "id": 151, "legend": { "alignAsTable": true, "avg": false, @@ -1763,6 +1766,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -1773,7 +1777,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"raftstore_.*\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_\\\\d+\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -1785,7 +1789,7 @@ }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"raftstore_.*\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_\\\\d+\"})", "hide": false, "interval": "", "intervalFactor": 2, @@ -1797,7 +1801,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Raft Store", + "title": "Storage Background (Small Tasks)", "tooltip": { "msResolution": false, "shared": true, @@ -1857,10 +1861,10 @@ "h": 7, "w": 12, "x": 12, - "y": 16 + "y": 46 }, "hiddenSeries": false, - "id": 153, + "id": 156, "legend": { "alignAsTable": true, "avg": false, @@ -1892,6 +1896,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -1902,7 +1907,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"snap_sender.*\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -1914,7 +1919,7 @@ }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"snap_sender.*\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"})", "hide": false, "interval": "", "intervalFactor": 2, @@ -1926,7 +1931,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Snapshot Sender", + "title": "Storage Background (Large Tasks)", "tooltip": { "msResolution": false, "shared": true, @@ -1972,7 +1977,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, - "description": "", + "description": "Involved when manually compacting the data.", "editable": true, "error": false, "fieldConfig": { @@ -1986,10 +1991,10 @@ "h": 7, "w": 12, "x": 0, - "y": 23 + "y": 53 }, "hiddenSeries": false, - "id": 151, + "id": 149, "legend": { "alignAsTable": true, "avg": false, @@ -2021,6 +2026,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -2031,7 +2037,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_\\\\d+\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"m_compact_pool\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -2043,7 +2049,7 @@ }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_\\\\d+\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"m_compact_pool\"})", "hide": false, "interval": "", "intervalFactor": 2, @@ -2055,7 +2061,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Storage Background (Small Tasks)", + "title": "Manual Compaction", "tooltip": { "msResolution": false, "shared": true, @@ -2115,10 +2121,10 @@ "h": 7, "w": 12, "x": 12, - "y": 23 + "y": 53 }, "hiddenSeries": false, - "id": 156, + "id": 159, "legend": { "alignAsTable": true, "avg": false, @@ -2150,6 +2156,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -2160,7 +2167,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"async_poller.*\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -2172,7 +2179,7 @@ }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"async_poller.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "hide": false, "interval": "", "intervalFactor": 2, @@ -2184,7 +2191,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Storage Background (Large Tasks)", + "title": "GRPC Async Server", "tooltip": { "msResolution": false, "shared": true, @@ -2230,7 +2237,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, - "description": "Involved when manually compacting the data.", + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -2244,10 +2251,10 @@ "h": 7, "w": 12, "x": 0, - "y": 30 + "y": 60 }, "hiddenSeries": false, - "id": 149, + "id": 161, "legend": { "alignAsTable": true, "avg": false, @@ -2279,6 +2286,7 @@ "alias": "Limit", "color": "#F2495C", "hideTooltip": true, + "legend": false, "linewidth": 2, "nullPointMode": "connected" } @@ -2289,7 +2297,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"m_compact_pool\"}[1m]))", + "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"GRPCComp.*\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -2301,7 +2309,7 @@ }, { "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"m_compact_pool\"})", + "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "hide": false, "interval": "", "intervalFactor": 2, @@ -2313,7 +2321,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Manual Compaction", + "title": "GRPC Async Client", "tooltip": { "msResolution": false, "shared": true, @@ -2351,32 +2359,42 @@ "align": false, "alignLevel": null } - }, + } + ], + "title": "Threads CPU", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 6, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "description": "", - "editable": true, - "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 30 + "x": 0, + "y": 3 }, "hiddenSeries": false, - "id": 159, + "id": 9, "legend": { "alignAsTable": true, "avg": false, @@ -2385,16 +2403,13 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -2403,48 +2418,25 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Limit", - "color": "#F2495C", - "hideTooltip": true, - "linewidth": 2, - "nullPointMode": "connected" - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"async_poller.*\"}[1m]))", + "expr": "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{name}} {{instance}}", - "refId": "A", - "step": 40 - }, - { - "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"async_poller.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Limit", - "refId": "B" + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GRPC Async Server", + "title": "Request QPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -2459,8 +2451,8 @@ }, "yaxes": [ { - "decimals": 1, - "format": "percentunit", + "decimals": null, + "format": "none", "label": null, "logBase": 1, "max": null, @@ -2468,12 +2460,12 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -2487,25 +2479,20 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "description": "", - "editable": true, - "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 37 + "x": 12, + "y": 3 }, "hiddenSeries": false, - "id": 161, + "id": 2, "legend": { "alignAsTable": true, "avg": false, @@ -2514,16 +2501,13 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -2532,48 +2516,25 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Limit", - "color": "#F2495C", - "hideTooltip": true, - "linewidth": 2, - "nullPointMode": "connected" - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"GRPCComp.*\"}[1m]))", + "expr": "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{name}} {{instance}}", - "refId": "A", - "step": 40 - }, - { - "exemplar": true, - "expr": "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Limit", - "refId": "B" + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GRPC Async Client", + "title": "Executor QPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -2588,8 +2549,7 @@ }, "yaxes": [ { - "decimals": 1, - "format": "percentunit", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -2597,34 +2557,19 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { "align": false, "alignLevel": null } - } - ], - "title": "Threads CPU", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 6, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -2635,26 +2580,26 @@ "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 3 + "y": 10 }, "hiddenSeries": false, - "id": 9, + "id": 11, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, @@ -2674,18 +2619,44 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}}", + "legendFormat": "999-{{type}}", "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "99-{{type}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "95-{{type}}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "80-{{type}}", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Request QPS", + "title": "Request Duration", "tooltip": { "shared": true, "sort": 0, @@ -2701,8 +2672,7 @@ }, "yaxes": [ { - "decimals": null, - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -2710,7 +2680,7 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2739,10 +2709,10 @@ "h": 7, "w": 12, "x": 12, - "y": 3 + "y": 10 }, "hiddenSeries": false, - "id": 2, + "id": 12, "legend": { "alignAsTable": true, "avg": false, @@ -2772,10 +2742,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{type}}", + "legendFormat": "{{reason}}", "refId": "A" } ], @@ -2783,7 +2753,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Executor QPS", + "title": "Error QPS", "tooltip": { "shared": true, "sort": 0, @@ -2836,10 +2806,10 @@ "h": 7, "w": 12, "x": 0, - "y": 10 + "y": 17 }, "hiddenSeries": false, - "id": 11, + "id": 13, "legend": { "alignAsTable": false, "avg": false, @@ -2869,34 +2839,29 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", - "interval": "", "intervalFactor": 1, "legendFormat": "999-{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", - "hide": true, "intervalFactor": 1, "legendFormat": "99-{{type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", - "hide": true, "intervalFactor": 1, "legendFormat": "95-{{type}}", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", - "hide": true, "intervalFactor": 1, "legendFormat": "80-{{type}}", "refId": "D" @@ -2906,7 +2871,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Request Duration", + "title": "Request Handle Duration", "tooltip": { "shared": true, "sort": 0, @@ -2959,10 +2924,10 @@ "h": 7, "w": 12, "x": 12, - "y": 10 + "y": 17 }, "hiddenSeries": false, - "id": 12, + "id": 14, "legend": { "alignAsTable": true, "avg": false, @@ -2992,227 +2957,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)", + "exemplar": true, + "expr": "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{reason}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Error QPS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "999-{{type}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99-{{type}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95-{{type}}", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "80-{{type}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request Handle Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{type}}", + "legendFormat": "{{type}}", "refId": "A" } ], @@ -6125,7 +5875,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of different kinds of read operations", + "description": "The number of currently opened file descriptors.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", "fieldConfig": { "defaults": {}, "overrides": [] @@ -6133,30 +5883,33 @@ "fill": 0, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, + "h": 7, + "w": 8, "x": 0, "y": 32 }, "hiddenSeries": false, - "id": 46, + "id": 50, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, - "hideZero": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -6171,43 +5924,52 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Page", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{instance}}", "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "exemplar": true, + "expr": "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "PageFile", - "refId": "C" + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "W-{{instance}}", + "refId": "B" }, { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "exemplar": true, + "expr": "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "File Descriptor", - "refId": "D" + "legendFormat": "R-{{instance}}", + "refId": "C" }, { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "exemplar": true, + "expr": "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "AIO", - "refId": "F" + "legendFormat": "RW-{{instance}}", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Disk Write OPS", + "title": "Opened File Count", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -6220,8 +5982,7 @@ }, "yaxes": [ { - "decimals": null, - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -6229,12 +5990,12 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -6248,7 +6009,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of different kinds of read operations", + "description": "The number of open file descriptors action.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", "fieldConfig": { "defaults": {}, "overrides": [] @@ -6256,30 +6017,33 @@ "fill": 0, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, - "x": 12, + "h": 7, + "w": 8, + "x": 8, "y": 32 }, "hiddenSeries": false, - "id": 47, + "id": 22, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, + "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -6294,43 +6058,32 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Page", - "refId": "A" - }, - { - "expr": "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "PageFile", - "refId": "C" - }, - { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "File Descriptor", - "refId": "D" + "legendFormat": "Open-{{instance}}", + "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIORead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "AIO", - "refId": "F" + "legendFormat": "OpenFail-{{instance}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Disk Read OPS", + "title": "File Open OPS", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -6343,7 +6096,6 @@ }, "yaxes": [ { - "decimals": null, "format": "ops", "label": null, "logBase": 1, @@ -6352,12 +6104,12 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -6371,35 +6123,31 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The flow of different kinds of write operations", + "description": "The number of fsync operations.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 + "h": 7, + "w": 8, + "x": 16, + "y": 32 }, - "height": "", "hiddenSeries": false, - "id": 60, + "id": 52, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, - "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", + "sideWidth": 250, + "sort": "max", "sortDesc": true, "total": false, "values": true @@ -6407,7 +6155,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -6416,52 +6164,27 @@ "pointradius": 5, "points": false, "renderer": "flot", - "repeatedByRow": true, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "File Descriptor", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Page", - "refId": "B" - }, - { - "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "PageBackGround", - "refId": "C" - }, - { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "AIO", - "refId": "D" + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write flow", + "title": "FSync OPS", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -6474,7 +6197,8 @@ }, "yaxes": [ { - "format": "binBps", + "decimals": null, + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -6482,11 +6206,11 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -6501,23 +6225,21 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The flow of different kinds of read operations", + "description": "The number of different kinds of read operations", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 40 + "x": 0, + "y": 39 }, - "height": "", "hiddenSeries": false, - "id": 59, + "id": 46, "legend": { "alignAsTable": true, "avg": false, @@ -6528,16 +6250,13 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -6546,48 +6265,46 @@ "pointradius": 5, "points": false, "renderer": "flot", - "repeatedByRow": true, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "File Descriptor", - "refId": "A", - "step": 10 + "legendFormat": "Page", + "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Page", - "refId": "B" + "hide": true, + "intervalFactor": 2, + "legendFormat": "PageFile", + "refId": "C" }, { - "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "PageBackGround", - "refId": "C" + "legendFormat": "File Descriptor", + "refId": "D" }, { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "AIO", - "refId": "D" + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Read flow", + "title": "Disk Write OPS", "tooltip": { "shared": true, "sort": 2, @@ -6603,7 +6320,8 @@ }, "yaxes": [ { - "format": "binBps", + "decimals": null, + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -6611,11 +6329,11 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -6630,25 +6348,27 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", + "description": "The number of different kinds of read operations", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 48 + "x": 12, + "y": 39 }, "hiddenSeries": false, - "id": 88, + "id": 47, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, + "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -6659,137 +6379,189 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/max_snapshot_lifetime/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "read_tasks-{{instance}}", - "refId": "I" + "intervalFactor": 2, + "legendFormat": "Page", + "refId": "A" }, { - "expr": "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "hide": true, - "intervalFactor": 1, - "legendFormat": "snapshot_list-{{instance}}", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "PageFile", + "refId": "C" }, { - "expr": "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "heatmap", - "hide": true, + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "format": "time_series", "intervalFactor": 1, - "legendFormat": "num_snapshot-{{instance}}", - "refId": "B" + "legendFormat": "File Descriptor", + "refId": "D" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIORead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", - "hide": true, "intervalFactor": 1, - "legendFormat": "read-{{instance}}", - "refId": "C" - }, - { - "expr": "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "read_raw-{{instance}}", - "refId": "D" - }, - { - "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "delta_merge-{{instance}}", - "refId": "E" - }, - { - "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "delta_compact-{{instance}}", - "refId": "J" - }, - { - "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "seg_merge-{{instance}}", + "legendFormat": "AIO", "refId": "F" - }, + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Read OPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "seg_split-{{instance}}", - "refId": "G" + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The flow of different kinds of write operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "height": "", + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeatedByRow": true, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "place_index-{{instance}}", - "refId": "H" + "hide": false, + "intervalFactor": 2, + "legendFormat": "File Descriptor", + "refId": "A", + "step": 10 }, { - "expr": "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "max_snapshot_lifetime-{{instance}}", - "refId": "K" + "legendFormat": "Page", + "refId": "B" }, { - "expr": "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", - "hide": true, "intervalFactor": 1, - "legendFormat": "max_snapshot_lifetime_stable-{{instance}}", - "refId": "L" + "legendFormat": "PageBackGround", + "refId": "C" }, { - "expr": "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", - "hide": true, + "interval": "", "intervalFactor": 1, - "legendFormat": "max_snapshot_lifetime_meta-{{instance}}", - "refId": "M" + "legendFormat": "AIO", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Read Snapshots", + "title": "Write flow", "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -6802,7 +6574,7 @@ }, "yaxes": [ { - "format": "short", + "format": "binBps", "label": null, "logBase": 1, "max": null, @@ -6810,11 +6582,11 @@ "show": true }, { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -6826,40 +6598,46 @@ { "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The current processing number of segments' background management", + "decimals": 1, + "description": "The flow of different kinds of read operations", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 46 }, + "height": "", "hiddenSeries": false, - "id": 67, + "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "hideEmpty": false, + "hideZero": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -6868,42 +6646,51 @@ "pointradius": 5, "points": false, "renderer": "flot", + "repeatedByRow": true, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "hide": false, - "intervalFactor": 1, - "legendFormat": "delta_merge-{{instance}}", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "File Descriptor", + "refId": "A", + "step": 10 }, { - "expr": "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "seg_split-{{instance}}", + "legendFormat": "Page", "refId": "B" }, { - "expr": "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "seg_merge-{{instance}}", + "legendFormat": "PageBackGround", "refId": "C" + }, + { + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AIO", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Current Data Management Tasks", + "title": "Read flow", "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -6916,8 +6703,7 @@ }, "yaxes": [ { - "decimals": 0, - "format": "short", + "format": "binBps", "label": null, "logBase": 1, "max": null, @@ -6925,11 +6711,11 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -6955,7 +6741,7 @@ "h": 8, "w": 12, "x": 0, - "y": 56 + "y": 54 }, "hiddenSeries": false, "id": 84, @@ -7055,7 +6841,7 @@ "h": 8, "w": 12, "x": 12, - "y": 56 + "y": 54 }, "hiddenSeries": false, "id": 86, @@ -7187,7 +6973,7 @@ "h": 8, "w": 12, "x": 0, - "y": 64 + "y": 62 }, "hiddenSeries": false, "id": 132, @@ -7305,42 +7091,46 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Information about schema of column file, to learn the memory usage of schema", + "description": "The current processing number of segments' background management", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 64 + "y": 62 }, "hiddenSeries": false, - "id": 168, + "id": 67, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "links": [], + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -7349,43 +7139,33 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"current_size\"}) by (instance)", - "interval": "", - "legendFormat": "current_size", - "queryType": "randomWalk", + "expr": "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "delta_merge-{{instance}}", "refId": "A" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit_count\"}[1m])) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "hit_count_ops", + "expr": "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "seg_split-{{instance}}", "refId": "B" }, { - "exemplar": true, - "expr": "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"still_used_when_evict\"}) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "still_used_when_evict", + "expr": "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "seg_merge-{{instance}}", "refId": "C" - }, - { - "exemplar": true, - "expr": "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"miss_count\"}) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "miss_count", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Schema of Column File", + "title": "Current Data Management Tasks", "tooltip": { "shared": true, "sort": 0, @@ -7401,15 +7181,16 @@ }, "yaxes": [ { + "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -7436,10 +7217,10 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 72 + "y": 70 }, "hiddenSeries": false, "id": 169, @@ -7577,129 +7358,163 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "S3 OPS", + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 72 + "y": 70 }, "hiddenSeries": false, - "id": 179, + "id": 88, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/max_snapshot_lifetime/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3PutObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "S3PutObject", - "refId": "B" + "legendFormat": "read_tasks-{{instance}}", + "refId": "I" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3GetObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "S3GetObject", + "expr": "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "snapshot_list-{{instance}}", "refId": "A" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3HeadObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "S3HeadObject", + "expr": "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "heatmap", + "hide": true, + "intervalFactor": 1, + "legendFormat": "num_snapshot-{{instance}}", + "refId": "B" + }, + { + "expr": "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "read-{{instance}}", "refId": "C" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3ListObjects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "S3ListObjects", + "expr": "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "read_raw-{{instance}}", "refId": "D" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "S3DeleteObject", + "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "delta_merge-{{instance}}", "refId": "E" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "S3CopyObject", + "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "delta_compact-{{instance}}", + "refId": "J" + }, + { + "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "seg_merge-{{instance}}", "refId": "F" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "S3CreateMultipartUpload", + "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "seg_split-{{instance}}", "refId": "G" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "S3UploadPart", + "expr": "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "place_index-{{instance}}", "refId": "H" }, { - "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", "hide": false, - "interval": "", - "legendFormat": "S3CompleteMultipartUpload", - "refId": "I" + "intervalFactor": 1, + "legendFormat": "max_snapshot_lifetime-{{instance}}", + "refId": "K" + }, + { + "expr": "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "max_snapshot_lifetime_stable-{{instance}}", + "refId": "L" + }, + { + "expr": "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "max_snapshot_lifetime_meta-{{instance}}", + "refId": "M" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "S3 OPS", + "title": "Read Snapshots", "tooltip": { "shared": true, "sort": 0, @@ -7715,8 +7530,7 @@ }, "yaxes": [ { - "decimals": null, - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -7724,11 +7538,11 @@ "show": true }, { - "format": "opm", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -7743,42 +7557,39 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "S3 read/write throughput", + "description": "Information about schema of column file, to learn the memory usage of schema", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 80 + "x": 12, + "y": 77 }, "hiddenSeries": false, - "id": 178, + "id": 168, "legend": { - "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -7788,28 +7599,42 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3WriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", - "format": "time_series", + "expr": "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"current_size\"}) by (instance)", + "interval": "", + "legendFormat": "current_size", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit_count\"}[1m])) by (instance)", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "S3WriteBytes", + "legendFormat": "hit_count_ops", "refId": "B" }, { "exemplar": true, - "expr": "sum(rate(tiflash_system_profile_event_S3ReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"still_used_when_evict\"}) by (instance)", "hide": false, "interval": "", - "legendFormat": "S3ReadBytes", - "refId": "A" + "legendFormat": "still_used_when_evict", + "refId": "C" + }, + { + "exemplar": true, + "expr": "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"miss_count\"}) by (instance)", + "hide": false, + "interval": "", + "legendFormat": "miss_count", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "S3 Bytes", + "title": "Schema of Column File", "tooltip": { "shared": true, "sort": 0, @@ -7825,151 +7650,19 @@ }, "yaxes": [ { - "$$hashKey": "object:263", - "decimals": null, - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "$$hashKey": "object:264", - "format": "opm", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "S3 Request Duration", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 80 - }, - "hiddenSeries": false, - "id": 180, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(1.0, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{type}}-max", - "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "hide": false, - "interval": "", - "legendFormat": "{{type}}-99", - "refId": "C" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "hide": false, - "interval": "", - "legendFormat": "{{type}}-90", - "refId": "D" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.80, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", - "hide": true, - "interval": "", - "legendFormat": "{{type}}-80", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "S3 Request Duration", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:195", - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:196", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, + "min": null, "show": true } ], @@ -11930,6 +11623,409 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "S3 read/write throughput", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 178, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3WriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "S3WriteBytes", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3ReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3ReadBytes", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "S3 Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "S3 OPS", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 179, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3PutObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "S3PutObject", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3GetObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3GetObject", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3HeadObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3HeadObject", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3ListObjects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3ListObjects", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3DeleteObject", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3CopyObject", + "refId": "F" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3CreateMultipartUpload", + "refId": "G" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3UploadPart", + "refId": "H" + }, + { + "exemplar": true, + "expr": "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "hide": false, + "interval": "", + "legendFormat": "S3CompleteMultipartUpload", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "S3 OPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "S3 Request Duration", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 180, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(1.0, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}-max", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}-99", + "refId": "C" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}-90", + "refId": "D" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.80, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "hide": true, + "interval": "", + "legendFormat": "{{type}}-80", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "S3 Request Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Disaggregated",