From b8e58ebcb741e3a921fc93352de72c5a20a5ba6f Mon Sep 17 00:00:00 2001 From: hehechen Date: Fri, 13 May 2022 20:44:38 +0800 Subject: [PATCH] add metrics for storage pool run mode (#4881) close pingcap/tiflash#4870 --- dbms/src/Common/CurrentMetrics.cpp | 7 +- dbms/src/Interpreters/Context.cpp | 3 +- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 16 +- dbms/src/Storages/DeltaMerge/StoragePool.h | 2 + .../Storages/Transaction/RegionPersister.cpp | 6 + metrics/grafana/tiflash_summary.json | 283 ++++++++++++++++++ 6 files changed, 314 insertions(+), 3 deletions(-) diff --git a/dbms/src/Common/CurrentMetrics.cpp b/dbms/src/Common/CurrentMetrics.cpp index 06173cc3cc9..8a2f111d882 100644 --- a/dbms/src/Common/CurrentMetrics.cpp +++ b/dbms/src/Common/CurrentMetrics.cpp @@ -78,7 +78,12 @@ M(IOLimiterPendingBgWriteReq) \ M(IOLimiterPendingFgWriteReq) \ M(IOLimiterPendingBgReadReq) \ - M(IOLimiterPendingFgReadReq) + M(IOLimiterPendingFgReadReq) \ + M(StoragePoolV2Only) \ + M(StoragePoolV3Only) \ + M(StoragePoolMixMode) \ + M(RegionPersisterRunMode) \ + M(GlobalStorageRunMode) namespace CurrentMetrics { diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index a2088483c20..1c74b5f402e 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -81,6 +81,7 @@ namespace CurrentMetrics { extern const Metric ContextLockWait; extern const Metric MemoryTrackingForMerges; +extern const Metric GlobalStorageRunMode; } // namespace CurrentMetrics @@ -1652,7 +1653,7 @@ bool Context::initializeGlobalStoragePoolIfNeed(const PathPool & path_pool) // GlobalStoragePool may be initialized many times in some test cases for restore. LOG_WARNING(shared->log, "GlobalStoragePool has already been initialized."); } - + CurrentMetrics::set(CurrentMetrics::GlobalStorageRunMode, static_cast(shared->storage_run_mode)); if (shared->storage_run_mode == PageStorageRunMode::MIX_MODE || shared->storage_run_mode == PageStorageRunMode::ONLY_V3) { try diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index f0c3db1e4e8..42be13c9ecf 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -19,6 +20,14 @@ #include #include + +namespace CurrentMetrics +{ +extern const Metric StoragePoolV2Only; +extern const Metric StoragePoolV3Only; +extern const Metric StoragePoolMixMode; +} // namespace CurrentMetrics + namespace DB { namespace ErrorCodes @@ -30,6 +39,7 @@ namespace FailPoints { extern const char force_set_dtfile_exist_when_acquire_id[]; } // namespace FailPoints + namespace DM { enum class StorageType @@ -159,6 +169,7 @@ StoragePool::StoragePool(Context & global_ctx, NamespaceId ns_id_, StoragePathPo , run_mode(global_ctx.getPageStorageRunMode()) , ns_id(ns_id_) , global_context(global_ctx) + , storage_pool_metrics(CurrentMetrics::StoragePoolV3Only, 0) { const auto & global_storage_pool = global_context.getGlobalStoragePool(); switch (run_mode) @@ -200,7 +211,6 @@ StoragePool::StoragePool(Context & global_ctx, NamespaceId ns_id_, StoragePathPo log_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, log_storage_v3); data_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, data_storage_v3); meta_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, meta_storage_v3); - break; } case PageStorageRunMode::MIX_MODE: @@ -303,6 +313,7 @@ PageStorageRunMode StoragePool::restore() max_data_page_id = data_max_ids[0]; max_meta_page_id = meta_max_ids[0]; + storage_pool_metrics = CurrentMetrics::Increment{CurrentMetrics::StoragePoolV2Only}; break; } case PageStorageRunMode::ONLY_V3: @@ -311,6 +322,7 @@ PageStorageRunMode StoragePool::restore() max_data_page_id = global_storage_pool->getDataMaxId(ns_id); max_meta_page_id = global_storage_pool->getMetaMaxId(ns_id); + storage_pool_metrics = CurrentMetrics::Increment{CurrentMetrics::StoragePoolV3Only}; break; } case PageStorageRunMode::MIX_MODE: @@ -368,12 +380,14 @@ PageStorageRunMode StoragePool::restore() max_meta_page_id = global_storage_pool->getMetaMaxId(ns_id); run_mode = PageStorageRunMode::ONLY_V3; + storage_pool_metrics = CurrentMetrics::Increment{CurrentMetrics::StoragePoolV3Only}; } else // Still running Mix Mode { max_log_page_id = std::max(v2_log_max_ids[0], global_storage_pool->getLogMaxId(ns_id)); max_data_page_id = std::max(v2_data_max_ids[0], global_storage_pool->getDataMaxId(ns_id)); max_meta_page_id = std::max(v2_meta_max_ids[0], global_storage_pool->getMetaMaxId(ns_id)); + storage_pool_metrics = CurrentMetrics::Increment{CurrentMetrics::StoragePoolMixMode}; } break; } diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 14dac73b667..2e3b3f563f5 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -235,6 +235,8 @@ class StoragePool : private boost::noncopyable std::atomic max_meta_page_id = 0; BackgroundProcessingPool::TaskHandle gc_handle = nullptr; + + CurrentMetrics::Increment storage_pool_metrics; }; struct StorageSnapshot : private boost::noncopyable diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index 8e6ed6821df..154112adfd4 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -28,6 +28,11 @@ #include +namespace CurrentMetrics +{ +extern const Metric RegionPersisterRunMode; +} + namespace DB { namespace ErrorCodes @@ -320,6 +325,7 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, } } + CurrentMetrics::set(CurrentMetrics::RegionPersisterRunMode, static_cast(run_mode)); LOG_FMT_INFO(log, "RegionPersister running. Current Run Mode is {}", static_cast(run_mode)); } diff --git a/metrics/grafana/tiflash_summary.json b/metrics/grafana/tiflash_summary.json index a1c75e7c04e..5b534e27f01 100644 --- a/metrics/grafana/tiflash_summary.json +++ b/metrics/grafana/tiflash_summary.json @@ -7390,6 +7390,289 @@ ], "title": "Rough Set Filter Rate Histogram", "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 119, + "panels": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "description": "The Global StoragePool and KVStore Runmode", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 11, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "decimals": 0, + "mappings": [ + { + "from": "", + "id": 1, + "text": "ONLY_V2", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "ONLY_V3", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 3, + "text": "MIX_MODE", + "to": "", + "type": 1, + "value": "3" + }, + { + "from": "", + "id": 4, + "text": " ", + "to": "", + "type": 1, + "value": "4" + }, + { + "from": "", + "id": 5, + "text": " ", + "to": "", + "type": 1, + "value": "5" + } + ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 126, + "links": [], + "options": { + "graph": {}, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "multi" + } + }, + "pluginVersion": "7.5.11", + "targets": [ + { + "exemplar": true, + "expr": "tiflash_system_current_metric_GlobalStorageRunMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}-GlobalRunMode", + "refId": "A", + "step": 10 + }, + { + "exemplar": false, + "expr": "tiflash_system_current_metric_RegionPersisterRunMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-KVStoreRunMode", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Global Runmode", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The StoragePool Runmode in DeltaMerge Storage", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 123, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(tiflash_system_current_metric_StoragePoolV2Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}-OnlyV2", + "refId": "A", + "step": 10 + }, + { + "exemplar": true, + "expr": "sum(tiflash_system_current_metric_StoragePoolV3Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-OnlyV3", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(tiflash_system_current_metric_StoragePoolMixMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-MixMode", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "StoragePool Runmode", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:650", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:651", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "StoragePool", + "type": "row" } ], "refresh": false,