From b0958e703b4a09d608506fd224bc00f7f69497aa Mon Sep 17 00:00:00 2001 From: Justin Huang Date: Tue, 8 Mar 2022 16:25:49 +0800 Subject: [PATCH] metrics: support multi-k8s in grafana dashboards (#4099) ref pingcap/tics#4129 --- metrics/grafana/tiflash_proxy_details.json | 565 +++++++++++---------- metrics/grafana/tiflash_proxy_summary.json | 69 ++- metrics/grafana/tiflash_summary.json | 336 ++++++------ 3 files changed, 512 insertions(+), 458 deletions(-) diff --git a/metrics/grafana/tiflash_proxy_details.json b/metrics/grafana/tiflash_proxy_details.json index 8d2a74c8c43..bc172a55f64 100644 --- a/metrics/grafana/tiflash_proxy_details.json +++ b/metrics/grafana/tiflash_proxy_details.json @@ -112,7 +112,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tiflash_proxy_process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}[1m])", + "expr": "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -209,7 +209,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_proxy_process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -306,7 +306,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{device}}", @@ -403,7 +403,7 @@ "steppedLine": false, "targets": [ { - "expr": "(time() - tiflash_proxy_process_start_time_seconds{tidb_cluster=\"$tidb_cluster\"})", + "expr": "(time() - tiflash_proxy_process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -505,7 +505,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -514,7 +514,7 @@ "step": 10 }, { - "expr": "delta(tiflash_proxy_tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[30s]) < -10", + "expr": "delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[30s]) < -10", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -611,7 +611,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -750,7 +750,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_critical_error_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(rate(tiflash_proxy_tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", @@ -855,7 +855,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "scheduler-{{instance}}", @@ -864,7 +864,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "channelfull-{{instance}}-{{type}}", @@ -873,7 +873,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "coprocessor-{{instance}}", @@ -882,7 +882,7 @@ "step": 4 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)", + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)", "format": "time_series", "intervalFactor": 2, "legendFormat": "stall-{{instance}}-{{db}}", @@ -1014,7 +1014,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_server_report_failure_msg_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance,store_id)", + "expr": "sum(rate(tiflash_proxy_tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance,store_id)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{type}} - to - {{store_id}}", @@ -1122,7 +1122,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[1m])) by (instance, status)", + "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[1m])) by (instance, status)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{status}}", @@ -1222,7 +1222,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[1m])) by (instance, stage)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[1m])) by (instance, stage)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{stage}}", @@ -1322,7 +1322,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, reason)", + "expr": "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, reason)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{reason}}", @@ -1422,7 +1422,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", @@ -1527,7 +1527,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(tiflash_proxy_tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[1m])) by (instance)", + "expr": "sum(delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1631,7 +1631,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_raftstore_leader_missing{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1744,7 +1744,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", + "expr": "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -1841,7 +1841,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1907,7 +1907,7 @@ "query": { "datasourceId": 1, "model": { - "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "intervalFactor": 2, "legendFormat": "{{instance}} - {{type}}", "metric": "", @@ -1983,7 +1983,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{type}}", @@ -2090,7 +2090,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_region_written_keys_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2222,7 +2222,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -2231,7 +2231,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -2240,7 +2240,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_region_size_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_region_size_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) ", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -2339,7 +2339,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -2438,7 +2438,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_region_written_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_bytes_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2602,7 +2602,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_region_written_keys_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_keys_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2684,7 +2684,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_region_written_keys_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2763,7 +2763,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2771,7 +2771,7 @@ "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}} 99", @@ -2863,14 +2863,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_server_request_batch_size_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_size_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}} avg", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}} 99", @@ -2977,7 +2977,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -3071,7 +3071,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -3167,7 +3167,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3262,7 +3262,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3357,7 +3357,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3366,21 +3366,21 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99% response", "refId": "B" }, { - "expr": "sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg request", "refId": "C" }, { - "expr": "sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg response", @@ -3473,7 +3473,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3482,7 +3482,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -3561,7 +3561,7 @@ "query": { "datasourceId": 1, "model": { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "tiflash_proxy_thread_cpu_seconds_total", @@ -3637,7 +3637,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3778,7 +3778,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3921,7 +3921,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4059,7 +4059,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4199,7 +4199,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unf_rd_pool.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unf_rd_pool.*\"}[1m])) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -4342,7 +4342,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - normal", @@ -4351,7 +4351,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - high", @@ -4360,7 +4360,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - low", @@ -4502,7 +4502,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[1m])) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -4511,7 +4511,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[1m])) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -4520,7 +4520,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[1m])) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -4626,7 +4626,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4739,7 +4739,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"ReadIndexWkr_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"ReadIndexWkr_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4833,7 +4833,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4931,7 +4931,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -5031,7 +5031,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -5040,7 +5040,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-region-worker", @@ -5148,7 +5148,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ type }}", @@ -5241,7 +5241,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ type }}", @@ -5334,7 +5334,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_pd_heartbeat_message_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ type }}", @@ -5427,7 +5427,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_pd_validate_peer_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ type }}", @@ -5537,7 +5537,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": " 99%", @@ -5546,7 +5546,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -5554,7 +5554,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) ", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -5649,7 +5649,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": " {{instance}}", @@ -5744,7 +5744,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": " 99%", @@ -5753,7 +5753,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -5761,7 +5761,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -5856,7 +5856,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} ", @@ -5946,21 +5946,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", "refId": "B" }, { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -6048,7 +6048,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -6158,7 +6158,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_ready_handled_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -6167,7 +6167,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "count", @@ -6264,7 +6264,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -6369,7 +6369,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_event_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -6489,7 +6489,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -6586,7 +6586,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_server_raft_message_flush_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -6681,7 +6681,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_server_raft_message_recv_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -6777,7 +6777,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -6874,7 +6874,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -6971,7 +6971,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_dropped_message_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -7080,7 +7080,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_proposal_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -7177,7 +7177,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index|batch\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index|batch\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -7275,7 +7275,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -7373,7 +7373,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -7469,7 +7469,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -7478,7 +7478,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -7486,7 +7486,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -7581,7 +7581,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -7675,7 +7675,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -7684,7 +7684,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -7692,7 +7692,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -7786,7 +7786,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -7878,7 +7878,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(rate(tiflash_proxy_tikv_raftstore_propose_log_size_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "avg(rate(tiflash_proxy_tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -7971,7 +7971,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_store_perf_context_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "store-{{type}}", @@ -7980,7 +7980,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_perf_context_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "apply-{{type}}", @@ -8092,7 +8092,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -8190,7 +8190,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_admin_cmd_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -8288,7 +8288,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_raftstore_check_split_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -8387,7 +8387,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -8494,7 +8494,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_multilevel_level_elapsed{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (level)", + "expr": "sum(rate(tiflash_proxy_tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (level)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{level}}", @@ -8582,7 +8582,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_multilevel_level0_chance{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}", + "expr": "tiflash_proxy_tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -8670,7 +8670,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(avg_over_time(tiflash_proxy_tikv_unified_read_pool_running_tasks{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(avg_over_time(tiflash_proxy_tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -8782,7 +8782,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_storage_command_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -8881,7 +8881,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[1m])) by (status)", + "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[1m])) by (status)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{status}}", @@ -8980,7 +8980,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -8988,7 +8988,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -8996,7 +8996,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -9094,7 +9094,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -9102,7 +9102,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -9110,7 +9110,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -9221,7 +9221,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (stage)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (stage)", "format": "time_series", "intervalFactor": 2, "legendFormat": "busy", @@ -9229,7 +9229,7 @@ "step": 20 }, { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (stage)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (stage)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{stage}}", @@ -9324,7 +9324,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_scheduler_writing_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -9422,7 +9422,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_commands_pri_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (priority)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (priority)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{priority}}", @@ -9556,7 +9556,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_scheduler_contex_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -9683,7 +9683,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "busy", @@ -9691,7 +9691,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (stage)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (stage)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{stage}}", @@ -9796,7 +9796,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -9805,7 +9805,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -9814,7 +9814,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -9921,7 +9921,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -9930,7 +9930,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -9939,7 +9939,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -10046,7 +10046,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -10055,7 +10055,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -10064,7 +10064,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -10171,7 +10171,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -10180,7 +10180,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -10189,7 +10189,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -10296,7 +10296,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[1m])) by (tag)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[1m])) by (tag)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tag}}", @@ -10403,7 +10403,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[1m])) by (tag)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[1m])) by (tag)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tag}}", @@ -10510,7 +10510,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[1m])) by (tag)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[1m])) by (tag)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tag}}", @@ -10617,7 +10617,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[1m])) by (tag)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[1m])) by (tag)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tag}}", @@ -10729,7 +10729,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(tiflash_proxy_tikv_raftstore_raft_sent_message_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m]))", + "expr": "sum(delta(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": " ", @@ -10825,7 +10825,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_send_snapshot_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "send", @@ -10833,7 +10833,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "apply", @@ -10841,7 +10841,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "generate", @@ -10937,7 +10937,7 @@ "steppedLine": true, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_raftstore_snapshot_traffic_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", + "expr": "sum(tiflash_proxy_tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -11034,7 +11034,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "size", @@ -11131,7 +11131,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_kv_count_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "count", @@ -11246,7 +11246,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_worker_handled_task_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", + "expr": "sum(rate(tiflash_proxy_tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", @@ -11345,7 +11345,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_worker_pending_task_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", + "expr": "sum(rate(tiflash_proxy_tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", @@ -11444,7 +11444,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_futurepool_handled_task_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", + "expr": "sum(rate(tiflash_proxy_tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", @@ -11543,7 +11543,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_futurepool_pending_task_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", + "expr": "sum(rate(tiflash_proxy_tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", @@ -11651,7 +11651,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_threads_state{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", + "expr": "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", @@ -11659,7 +11659,7 @@ "step": 4 }, { - "expr": "sum(tiflash_proxy_threads_state{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-total", @@ -11750,7 +11750,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_threads_io_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (name, io) > 1024", + "expr": "sum(rate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (name, io) > 1024", "format": "time_series", "hide": false, "interval": "", @@ -11844,7 +11844,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_voluntary_context_switches{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (instance, name) > 200", + "expr": "sum(rate(tiflash_proxy_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (instance, name) > 200", "format": "time_series", "hide": false, "interval": "", @@ -11938,7 +11938,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_nonvoluntary_context_switches{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (instance, name) > 50", + "expr": "sum(rate(tiflash_proxy_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (instance, name) > 50", "format": "time_series", "hide": false, "interval": "", @@ -12055,7 +12055,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "memtable", @@ -12064,7 +12064,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "block_cache", @@ -12073,7 +12073,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_get_served{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "l0", @@ -12081,7 +12081,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_get_served{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "l1", @@ -12089,7 +12089,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_get_served{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "l2_and_up", @@ -12189,7 +12189,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_get_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -12197,7 +12197,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_get_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -12205,7 +12205,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_get_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -12213,7 +12213,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_get_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -12313,7 +12313,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "seek", @@ -12322,7 +12322,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "seek_found", @@ -12331,7 +12331,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "next", @@ -12340,7 +12340,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "next_found", @@ -12349,7 +12349,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "prev", @@ -12358,7 +12358,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "prev_found", @@ -12459,7 +12459,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_seek_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -12467,7 +12467,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -12475,7 +12475,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -12483,7 +12483,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -12583,7 +12583,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "done", @@ -12591,7 +12591,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "timeout", @@ -12599,7 +12599,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "with_wal", @@ -12699,7 +12699,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -12707,7 +12707,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -12715,7 +12715,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -12723,7 +12723,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -12823,7 +12823,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -12831,7 +12831,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -12839,7 +12839,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -12847,7 +12847,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -12947,7 +12947,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "sync", @@ -13049,7 +13049,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -13057,7 +13057,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -13065,7 +13065,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -13073,7 +13073,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -13173,7 +13173,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_event_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -13274,7 +13274,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_compaction_time{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -13283,7 +13283,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_compaction_time{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -13291,7 +13291,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_compaction_time{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -13299,7 +13299,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_compaction_time{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -13399,7 +13399,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_sst_read_micros{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -13408,7 +13408,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -13417,7 +13417,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -13426,7 +13426,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -13527,7 +13527,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -13536,7 +13536,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -13545,7 +13545,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -13554,7 +13554,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -13655,7 +13655,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_proxy_tikv_engine_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", + "expr": "avg(tiflash_proxy_tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cf}}", @@ -13755,7 +13755,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[1m])))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[1m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "hit", @@ -13855,7 +13855,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))", + "expr": "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{cf}}", @@ -13956,7 +13956,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[1m])))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[1m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "all", @@ -13965,7 +13965,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "data", @@ -13974,7 +13974,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "filter", @@ -13983,7 +13983,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "index", @@ -13992,7 +13992,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "bloom prefix", @@ -14094,7 +14094,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14104,7 +14104,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14114,7 +14114,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14125,7 +14125,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14136,7 +14136,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14147,7 +14147,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14158,7 +14158,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14261,7 +14261,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "total_add", @@ -14270,7 +14270,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "data_add", @@ -14279,7 +14279,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "filter_add", @@ -14288,7 +14288,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "index_add", @@ -14297,7 +14297,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "add_failures", @@ -14399,7 +14399,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14409,7 +14409,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14419,7 +14419,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14522,7 +14522,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_engine_estimate_num_keys{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)", + "expr": "sum(tiflash_proxy_tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -14625,7 +14625,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14635,7 +14635,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", "format": "time_series", "hide": false, "interval": "", @@ -14738,7 +14738,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_bytes_per_read{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -14746,7 +14746,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_read{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -14754,7 +14754,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_read{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -14762,7 +14762,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_read{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -14863,7 +14863,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -14872,7 +14872,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -14974,7 +14974,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", + "expr": "max(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -14982,7 +14982,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", + "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", @@ -14990,7 +14990,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", + "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -14998,7 +14998,7 @@ "step": 10 }, { - "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})", + "expr": "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -15098,7 +15098,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15107,7 +15107,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15116,7 +15116,7 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15217,7 +15217,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_pending_compaction_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (cf)", + "expr": "sum(rate(tiflash_proxy_tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (cf)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15319,7 +15319,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15421,7 +15421,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_proxy_tikv_engine_compression_ratio{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)", + "expr": "avg(tiflash_proxy_tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15523,7 +15523,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_engine_num_snapshots{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", + "expr": "tiflash_proxy_tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15625,7 +15625,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_engine_oldest_snapshot_duration{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", + "expr": "tiflash_proxy_tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -15725,7 +15725,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_proxy_tikv_engine_num_files_at_level{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)", + "expr": "avg(tiflash_proxy_tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)", "format": "time_series", "intervalFactor": 2, "legendFormat": "cf-{{cf}}, level-{{level}}", @@ -15822,14 +15822,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", "refId": "A" }, { - "expr": "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "average", @@ -15927,7 +15927,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_engine_stall_conditions_changed{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", + "expr": "tiflash_proxy_tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{cf}}-{{type}}", @@ -16025,7 +16025,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", + "expr": "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -16126,7 +16126,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -16232,7 +16232,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_encryption_data_key_storage_total{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tiflash_proxy_tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -16322,7 +16322,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_encryption_file_num{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tiflash_proxy_tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -16410,7 +16410,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_encryption_is_initialized{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tiflash_proxy_tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -16500,7 +16500,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_encryption_meta_file_size_bytes{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tiflash_proxy_tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{name}}-{{instance}}", @@ -16588,21 +16588,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation))", + "expr": "histogram_quantile(1, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation))", "format": "time_series", "intervalFactor": 2, "legendFormat": "max-{{type}}-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%-{{type}}-{{operation}}", "refId": "B" }, { - "expr": "sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation) / sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation)", + "expr": "sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation) / sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, operation)", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg-{{type}}-{{operation}}", @@ -16663,25 +16663,40 @@ "list": [ { "allValue": null, - "current": { - }, + "current": { }, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [ ], + "query": "label_values(tiflash_proxy_tikv_engine_block_cache_size_bytes, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", - "options": [ - - ], - "query": "label_values(tiflash_proxy_tikv_engine_block_cache_size_bytes, tidb_cluster)", + "options": [ ], + "query": "label_values(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", - "tags": [ - - ], + "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false @@ -16697,7 +16712,7 @@ "multi": true, "name": "db", "options": [], - "query": "label_values(tiflash_proxy_tikv_engine_block_cache_size_bytes{tidb_cluster=\"$tidb_cluster\"}, db)", + "query": "label_values(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -16719,7 +16734,7 @@ "multi": true, "name": "command", "options": [], - "query": "label_values(tiflash_proxy_tikv_storage_command_total{tidb_cluster=\"$tidb_cluster\"}, type)", + "query": "label_values(tiflash_proxy_tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, type)", "refresh": 1, "regex": "prewrite|commit|rollback", "skipUrlSync": false, @@ -16741,7 +16756,7 @@ "multi": false, "name": "instance", "options": [], - "query": "label_values(tiflash_proxy_tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}, instance)", + "query": "label_values(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 1, "regex": "", "skipUrlSync": false, diff --git a/metrics/grafana/tiflash_proxy_summary.json b/metrics/grafana/tiflash_proxy_summary.json index 1b07ab0f3c6..ab1d141c9f8 100644 --- a/metrics/grafana/tiflash_proxy_summary.json +++ b/metrics/grafana/tiflash_proxy_summary.json @@ -106,7 +106,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{device}}", @@ -201,7 +201,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -313,7 +313,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "scheduler-{{instance}}", @@ -322,7 +322,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "channelfull-{{instance}}-{{type}}", @@ -331,7 +331,7 @@ "step": 4 }, { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\"}) by (instance)", + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "stall-{{instance}}", @@ -434,7 +434,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_raftstore_leader_missing{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_proxy_tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -545,7 +545,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_proxy_tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", + "expr": "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -626,7 +626,7 @@ "query": { "datasourceId": 1, "model": { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "tiflash_proxy_thread_cpu_seconds_total", @@ -702,7 +702,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -808,7 +808,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -921,7 +921,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"split_check\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"split_check\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1021,7 +1021,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1115,7 +1115,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1213,7 +1213,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1222,7 +1222,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-region-worker", @@ -1328,7 +1328,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ type }}", @@ -1419,7 +1419,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ type }}", @@ -1482,25 +1482,40 @@ "list": [ { "allValue": null, - "current": { - }, + "current": { }, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [ ], + "query": "label_values(tiflash_proxy_tikv_engine_size_bytes, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", - "options": [ - - ], - "query": "label_values(tiflash_proxy_tikv_engine_size_bytes, tidb_cluster)", + "options": [ ], + "query": "label_values(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", - "tags": [ - - ], + "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false @@ -1516,7 +1531,7 @@ "multi": true, "name": "instance", "options": [], - "query": "label_values(tiflash_proxy_tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}, instance)", + "query": "label_values(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 1, "regex": "", "skipUrlSync": false, diff --git a/metrics/grafana/tiflash_summary.json b/metrics/grafana/tiflash_summary.json index 0c440a3ab54..59b6ab9053e 100644 --- a/metrics/grafana/tiflash_summary.json +++ b/metrics/grafana/tiflash_summary.json @@ -112,7 +112,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_system_current_metric_StoreSizeUsed{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_system_current_metric_StoreSizeUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -209,7 +209,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_system_current_metric_StoreSizeAvailable{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_system_current_metric_StoreSizeAvailable{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -306,7 +306,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_system_current_metric_StoreSizeCapacity{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_system_current_metric_StoreSizeCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -407,7 +407,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_system_asynchronous_metric_Uptime{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_asynchronous_metric_Uptime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -498,7 +498,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_retained{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", + "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_retained{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -506,7 +506,7 @@ "refId": "A" }, { - "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", + "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -514,7 +514,7 @@ "refId": "B" }, { - "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_resident{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", + "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_resident{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -522,7 +522,7 @@ "refId": "C" }, { - "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", + "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -530,7 +530,7 @@ "refId": "D" }, { - "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_active{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", + "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_active{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -538,7 +538,7 @@ "refId": "E" }, { - "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", + "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -546,7 +546,7 @@ "refId": "F" }, { - "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", + "expr": "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -554,7 +554,7 @@ "refId": "G" }, { - "expr": "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"})", + "expr": "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -562,7 +562,7 @@ "refId": "H" }, { - "expr": "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"})", + "expr": "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -570,7 +570,7 @@ "refId": "I" }, { - "expr": "sum(tiflash_system_asynchronous_metric_mmap_alive{tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"})", + "expr": "sum(tiflash_system_asynchronous_metric_mmap_alive{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"})", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -578,7 +578,7 @@ "refId": "J" }, { - "expr": "tiflash_proxy_process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}", + "expr": "tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -675,7 +675,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tiflash_proxy_process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}[1m])", + "expr": "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -767,7 +767,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_FileFSync{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -858,14 +858,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_FileOpen{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Newly Open-{{instance}}", "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_FileOpenFailed{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Newly Open Failed-{{instance}}", @@ -955,7 +955,7 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_process_open_fds{tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}", + "expr": "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -963,7 +963,7 @@ "refId": "A" }, { - "expr": "sum(tiflash_system_current_metric_OpenFileForWrite{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -971,7 +971,7 @@ "refId": "B" }, { - "expr": "sum(tiflash_system_current_metric_OpenFileForRead{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -979,7 +979,7 @@ "refId": "C" }, { - "expr": "sum(tiflash_system_current_metric_OpenFileForReadWrite{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1082,7 +1082,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_coprocessor_request_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", @@ -1170,7 +1170,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_coprocessor_executor_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", @@ -1257,28 +1257,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "999", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "80", @@ -1365,7 +1365,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_coprocessor_request_error{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)", + "expr": "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{reason}}", @@ -1452,28 +1452,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "999", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "80", @@ -1560,7 +1560,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_coprocessor_response_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -1645,28 +1645,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "999-{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99-{{type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95-{{type}}", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "80-{{type}}", @@ -1753,7 +1753,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_coprocessor_handling_request_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", + "expr": "sum(tiflash_coprocessor_handling_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", @@ -1852,7 +1852,7 @@ "targets": [ { "exemplar": true, - "expr": "max(tiflash_thread_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)", + "expr": "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1951,7 +1951,7 @@ "targets": [ { "exemplar": true, - "expr": "max(tiflash_thread_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)", + "expr": "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -2050,7 +2050,7 @@ "targets": [ { "exemplar": true, - "expr": "max(tiflash_thread_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)", + "expr": "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -2149,7 +2149,7 @@ "targets": [ { "exemplar": true, - "expr": "max(tiflash_thread_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)", + "expr": "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -2252,7 +2252,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_schema_version{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_schema_version{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -2341,14 +2341,14 @@ "steppedLine": false, "targets": [ { - "expr": "avg(increase(tiflash_schema_apply_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "avg(increase(tiflash_schema_apply_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "A" }, { - "expr": "avg(increase(tiflash_schema_trigger_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "triggle-by-{{type}}", @@ -2437,21 +2437,21 @@ "steppedLine": false, "targets": [ { - "expr": "avg(increase(tiflash_schema_internal_ddl_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "A" }, { - "expr": "sum(increase(tiflash_schema_internal_ddl_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "total", "refId": "B" }, { - "expr": "sum(increase(tiflash_schema_internal_ddl_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance)", + "expr": "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -2459,7 +2459,7 @@ "refId": "C" }, { - "expr": "sum(increase(tiflash_schema_internal_ddl_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -2554,35 +2554,35 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "999", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "80", "refId": "D" }, { - "expr": "sum(tiflash_schema_applying{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_schema_applying{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "applying-{{instance}}", @@ -2700,7 +2700,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tiflash_storage_command_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -2708,7 +2708,7 @@ "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_DMWriteBlock{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2817,7 +2817,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_storage_write_amplification{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "max(tiflash_storage_write_amplification{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2825,7 +2825,7 @@ "refId": "A" }, { - "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", + "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2833,7 +2833,7 @@ "refId": "B" }, { - "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m])) / (rate(tiflash_system_profile_event_DMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m]))) by (instance)", + "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m])) / (rate(tiflash_system_profile_event_DMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[10m]))) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -2841,7 +2841,7 @@ "refId": "C" }, { - "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m])) / (rate(tiflash_system_profile_event_DMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m]))) by (instance)", + "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m])) / (rate(tiflash_system_profile_event_DMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30m]))) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -2849,7 +2849,7 @@ "refId": "D" }, { - "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", + "expr": "sum((rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -2857,7 +2857,7 @@ "refId": "E" }, { - "expr": "sum(rate(tiflash_system_profile_event_DMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_DMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -2957,7 +2957,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_storage_read_tasks_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_storage_read_tasks_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -3062,7 +3062,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_storage_subtask_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\"delta_merge|delta_merge_fg|delta_merge_bg_gc|seg_merge|seg_split|seg_split_fg\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\"delta_merge|delta_merge_fg|delta_merge_bg_gc|seg_merge|seg_split|seg_split_fg\"}[1m])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3070,7 +3070,7 @@ "refId": "A" }, { - "expr": "sum(increase(tiflash_storage_subtask_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"delta_merge|delta_merge_fg|delta_merge_bg_gc|seg_merge|seg_split|seg_split_fg\"}[1m])) by (type)", + "expr": "sum(increase(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"delta_merge|delta_merge_fg|delta_merge_bg_gc|seg_merge|seg_split|seg_split_fg\"}[1m])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3181,7 +3181,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", + "expr": "histogram_quantile(1, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3189,7 +3189,7 @@ "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -3197,7 +3197,7 @@ "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -3205,7 +3205,7 @@ "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -3304,7 +3304,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tiflash_storage_page_gc_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(increase(tiflash_storage_page_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", @@ -3390,7 +3390,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_storage_page_gc_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -3475,14 +3475,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Page", "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_PSMWritePages{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -3490,14 +3490,14 @@ "refId": "C" }, { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "File Descriptor", "refId": "D" }, { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "AIO", @@ -3598,14 +3598,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Page", "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_PSMReadPages{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -3613,14 +3613,14 @@ "refId": "C" }, { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "File Descriptor", "refId": "D" }, { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIORead{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIORead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "AIO", @@ -3727,7 +3727,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -3736,14 +3736,14 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_system_profile_event_PSMWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Page", "refId": "B" }, { - "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3850,7 +3850,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -3859,14 +3859,14 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_system_profile_event_PSMReadBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Page", "refId": "B" }, { - "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "AIO", @@ -3969,14 +3969,14 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_system_current_metric_DT_SegmentReadTasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "read_tasks-{{instance}}", "refId": "I" }, { - "expr": "tiflash_system_current_metric_PSMVCCSnapshotsList{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -3984,7 +3984,7 @@ "refId": "A" }, { - "expr": "tiflash_system_current_metric_PSMVCCNumSnapshots{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "heatmap", "hide": true, "intervalFactor": 1, @@ -3992,7 +3992,7 @@ "refId": "B" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfRead{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4000,7 +4000,7 @@ "refId": "C" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfReadRaw{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4008,7 +4008,7 @@ "refId": "D" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4016,7 +4016,7 @@ "refId": "E" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4024,7 +4024,7 @@ "refId": "J" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4032,7 +4032,7 @@ "refId": "F" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4040,7 +4040,7 @@ "refId": "G" }, { - "expr": "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4048,7 +4048,7 @@ "refId": "H" }, { - "expr": "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4056,7 +4056,7 @@ "refId": "K" }, { - "expr": "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4064,7 +4064,7 @@ "refId": "L" }, { - "expr": "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4164,7 +4164,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_system_current_metric_DT_DeltaMerge{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4172,14 +4172,14 @@ "refId": "A" }, { - "expr": "avg(tiflash_system_current_metric_DT_SegmentSplit{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "seg_split-{{instance}}", "refId": "B" }, { - "expr": "avg(tiflash_system_current_metric_DT_SegmentMerge{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "seg_merge-{{instance}}", @@ -4278,7 +4278,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_storage_io_limiter{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_storage_io_limiter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -4378,7 +4378,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -4387,7 +4387,7 @@ }, { "exemplar": true, - "expr": "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "hide": false, "interval": "", "legendFormat": "bgwrite-{{instance}}", @@ -4395,7 +4395,7 @@ }, { "exemplar": true, - "expr": "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "hide": false, "interval": "", "legendFormat": "fgwrite-{{instance}}", @@ -4403,7 +4403,7 @@ }, { "exemplar": true, - "expr": "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "hide": false, "interval": "", "legendFormat": "bgread-{{instance}}", @@ -4411,7 +4411,7 @@ }, { "exemplar": true, - "expr": "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "hide": false, "interval": "", "legendFormat": "fgread-{{instance}}", @@ -4528,7 +4528,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_storage_throughput_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[1m]))", + "expr": "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4537,21 +4537,21 @@ "step": 10 }, { - "expr": "sum(rate(tiflash_storage_throughput_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"}[1m]))", + "expr": "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "throughput_delta-management", "refId": "B" }, { - "expr": "sum(tiflash_storage_throughput_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"})", + "expr": "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "total_write", "refId": "C" }, { - "expr": "sum(tiflash_storage_throughput_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"})", + "expr": "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "total_delta-management", @@ -4644,7 +4644,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, instance))", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -4652,7 +4652,7 @@ "refId": "B" }, { - "expr": "histogram_quantile(1, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, instance))", + "expr": "histogram_quantile(1, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "max-{{type}}-{{instance}}", @@ -4753,7 +4753,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_storage_throughput_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4848,7 +4848,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_system_profile_event_DMWriteBlock{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4856,7 +4856,7 @@ "refId": "C" }, { - "expr": "sum(increase(tiflash_storage_command_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", + "expr": "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4959,7 +4959,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_raft_read_index_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_raft_read_index_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -5047,28 +5047,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "max", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "80", @@ -5162,35 +5162,35 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "max", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "80", "refId": "D" }, { - "expr": "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-timeout", @@ -5281,7 +5281,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "expr": "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -5376,7 +5376,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5387,7 +5387,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5397,7 +5397,7 @@ }, { "exemplar": true, - "expr": "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) ", + "expr": "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) ", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5407,7 +5407,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5417,7 +5417,7 @@ }, { "exemplar": true, - "expr": "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[1m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[1m])) ", + "expr": "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[1m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[1m])) ", "format": "time_series", "hide": false, "interval": "", @@ -5502,7 +5502,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -5568,7 +5568,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -5634,7 +5634,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -5700,7 +5700,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_flush\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_flush\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -5781,7 +5781,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_raft_process_keys{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tiflash_raft_process_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -5863,7 +5863,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_command_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"ingest_sst\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"ingest_sst\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -5929,7 +5929,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -5995,7 +5995,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -6061,7 +6061,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(delta(tiflash_raft_upstream_latency_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -6139,7 +6139,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_upstream_latency_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": " 100%", @@ -6148,7 +6148,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": " 99%", @@ -6157,7 +6157,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", @@ -6165,7 +6165,7 @@ "step": 4 }, { - "expr": "sum(rate(tiflash_raft_upstream_latency_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_raft_upstream_latency_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) ", + "expr": "sum(rate(tiflash_raft_upstream_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_raft_upstream_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -6292,7 +6292,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))) by (instance)", + "expr": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -6300,7 +6300,7 @@ "refId": "B" }, { - "expr": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", + "expr": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -6308,7 +6308,7 @@ "refId": "C" }, { - "expr": "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "hide": true, "instant": false, @@ -6317,7 +6317,7 @@ "refId": "A" }, { - "expr": "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "hide": true, "instant": false, @@ -6326,7 +6326,7 @@ "refId": "D" }, { - "expr": "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", + "expr": "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -6413,7 +6413,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", + "expr": "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", @@ -6456,6 +6456,30 @@ "tags": [], "templating": { "list": [ + { + "allValue": null, + "current": {}, + "datasource": "${DS_TEST-CLUSTER}", + "definition": "", + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [], + "query": "label_values(tiflash_system_profile_event_Query, k8s_cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": {}, @@ -6469,7 +6493,7 @@ "multi": false, "name": "tidb_cluster", "options": [], - "query": "label_values(tiflash_system_profile_event_Query, tidb_cluster)", + "query": "label_values(tiflash_system_profile_event_Query{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -6484,7 +6508,7 @@ "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", - "definition": "label_values(tiflash_system_profile_event_Query{tidb_cluster=\"$tidb_cluster\"}, instance)", + "definition": "label_values(tiflash_system_profile_event_Query{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "description": null, "error": null, "hide": 0, @@ -6493,7 +6517,7 @@ "multi": true, "name": "instance", "options": [], - "query": "label_values(tiflash_system_profile_event_Query{tidb_cluster=\"$tidb_cluster\"}, instance)", + "query": "label_values(tiflash_system_profile_event_Query{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 1, "regex": "", "skipUrlSync": false,