diff --git a/client/client.go b/client/client.go index 19d8f808934d..bc03c24dd082 100644 --- a/client/client.go +++ b/client/client.go @@ -324,8 +324,8 @@ const ( updateMemberTimeout = time.Second // Use a shorter timeout to recover faster from network isolation. tsLoopDCCheckInterval = time.Minute defaultMaxTSOBatchSize = 10000 // should be higher if client is sending requests in burst - retryInterval = time.Second - maxRetryTimes = 5 + retryInterval = 500 * time.Millisecond + maxRetryTimes = 6 ) // LeaderHealthCheckInterval might be changed in the unit to shorten the testing time. @@ -698,12 +698,11 @@ func (c *client) handleDispatcher( dc string, tbc *tsoBatchController) { var ( - retryTimeConsuming time.Duration - err error - streamAddr string - stream pdpb.PD_TsoClient - streamCtx context.Context - cancel context.CancelFunc + err error + streamAddr string + stream pdpb.PD_TsoClient + streamCtx context.Context + cancel context.CancelFunc // addr -> connectionContext connectionCtxs sync.Map opts []opentracing.StartSpanOption @@ -760,6 +759,7 @@ func (c *client) handleDispatcher( } // Loop through each batch of TSO requests and send them for processing. + streamLoopTimer := time.NewTimer(c.option.timeout) tsoBatchLoop: for { select { @@ -782,6 +782,7 @@ tsoBatchLoop: if maxBatchWaitInterval >= 0 { tbc.adjustBestBatchSize() } + streamLoopTimer.Reset(c.option.timeout) // Choose a stream to send the TSO gRPC request. streamChoosingLoop: for { @@ -792,24 +793,22 @@ tsoBatchLoop: // Check stream and retry if necessary. if stream == nil { log.Info("[pd] tso stream is not ready", zap.String("dc", dc)) - c.updateConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) - if retryTimeConsuming >= c.option.timeout { - err = errs.ErrClientCreateTSOStream.FastGenByArgs("retry timeout") - log.Error("[pd] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) - c.ScheduleCheckLeader() - c.finishTSORequest(tbc.getCollectedRequests(), 0, 0, 0, errors.WithStack(err)) - retryTimeConsuming = 0 - continue tsoBatchLoop + if c.updateConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) { + continue streamChoosingLoop } select { case <-dispatcherCtx.Done(): return - case <-time.After(time.Second): - retryTimeConsuming += time.Second - continue + case <-streamLoopTimer.C: + err = errs.ErrClientCreateTSOStream.FastGenByArgs(errs.RetryTimeoutErr) + log.Error("[pd] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) + c.ScheduleCheckLeader() + c.finishTSORequest(tbc.getCollectedRequests(), 0, 0, 0, errors.WithStack(err)) + continue tsoBatchLoop + case <-time.After(retryInterval): + continue streamChoosingLoop } } - retryTimeConsuming = 0 select { case <-streamCtx.Done(): log.Info("[pd] tso stream is canceled", zap.String("dc", dc), zap.String("stream-addr", streamAddr)) @@ -903,7 +902,7 @@ type connectionContext struct { cancel context.CancelFunc } -func (c *client) updateConnectionCtxs(updaterCtx context.Context, dc string, connectionCtxs *sync.Map) { +func (c *client) updateConnectionCtxs(updaterCtx context.Context, dc string, connectionCtxs *sync.Map) bool { // Normal connection creating, it will be affected by the `enableForwarding`. createTSOConnection := c.tryConnect if c.allowTSOFollowerProxy(dc) { @@ -911,7 +910,9 @@ func (c *client) updateConnectionCtxs(updaterCtx context.Context, dc string, con } if err := createTSOConnection(updaterCtx, dc, connectionCtxs); err != nil { log.Error("[pd] update connection contexts failed", zap.String("dc", dc), errs.ZapError(err)) + return false } + return true } // tryConnect will try to connect to the TSO allocator leader. If the connection becomes unreachable @@ -927,6 +928,8 @@ func (c *client) tryConnect( networkErrNum uint64 err error stream pdpb.PD_TsoClient + url string + cc *grpc.ClientConn ) updateAndClear := func(newAddr string, connectionCtx *connectionContext) { if cc, loaded := connectionCtxs.LoadOrStore(newAddr, connectionCtx); loaded { @@ -942,9 +945,11 @@ func (c *client) tryConnect( return true }) } - cc, url := c.getAllocatorClientConnByDCLocation(dc) // retry several times before falling back to the follower when the network problem happens + for i := 0; i < maxRetryTimes; i++ { + c.ScheduleCheckLeader() + cc, url = c.getAllocatorClientConnByDCLocation(dc) cctx, cancel := context.WithCancel(dispatcherCtx) stream, err = c.createTsoStream(cctx, cancel, pdpb.NewPDClient(cc)) failpoint.Inject("unreachableNetwork", func() { diff --git a/client/errs/errno.go b/client/errs/errno.go index ec49a46a7be0..14bcb9330467 100644 --- a/client/errs/errno.go +++ b/client/errs/errno.go @@ -21,6 +21,7 @@ const ( NotLeaderErr = "is not leader" // MismatchLeaderErr indicates the the non-leader member received the requests which should be received by leader. MismatchLeaderErr = "mismatch leader id" + RetryTimeoutErr = "retry timeout" ) // client errors diff --git a/client/go.mod b/client/go.mod index bc3b9f8d2c5d..4401e88b95e8 100644 --- a/client/go.mod +++ b/client/go.mod @@ -7,7 +7,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad - github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee + github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 go.uber.org/goleak v1.1.11 diff --git a/client/go.sum b/client/go.sum index 89398c56263f..f56e7a130b37 100644 --- a/client/go.sum +++ b/client/go.sum @@ -106,8 +106,8 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad h1:lGKxsEwdE0pVXzHYD1SQ1vfa3t/bFVU/latrQz8b/w0= github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/go.mod b/go.mod index b7359e26d70a..dc013b203dbc 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,8 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad - github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 + github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a + github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-dashboard v0.0.0-20220728104842-3743e533b594 github.com/prometheus/client_golang v1.1.0 diff --git a/go.sum b/go.sum index 6d3b46f03354..91add432cacb 100644 --- a/go.sum +++ b/go.sum @@ -417,13 +417,14 @@ github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMt github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad h1:lGKxsEwdE0pVXzHYD1SQ1vfa3t/bFVU/latrQz8b/w0= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a h1:McYxPhA8SHqfUtLfQHHN0fQl4dy93IkhlX4Pp2MKIFA= +github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= -github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 h1:SvWCbCPh1YeHd9yQLksvJYAgft6wLTY1aNG81tpyscQ= github.com/pingcap/log v0.0.0-20210906054005-afc726e70354/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g= github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM= github.com/pingcap/tidb-dashboard v0.0.0-20220728104842-3743e533b594 h1:kL1CW5qsn459kHZ2YoBYb+YOSWjSlshk55YP/XNQNWo= diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 38b0fc714056..39a7a6ab1a92 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -3941,7 +3941,7 @@ "format": "time_series", "hide": true, "intervalFactor": 1, - "legendFormat": "tolerant-resource-{{source}}-{{target}}", + "legendFormat": "tolerant-resource", "refId": "E" } ], @@ -8176,7 +8176,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Completed commands rate", + "title": "gRPC Completed commands rate", "tooltip": { "msResolution": false, "shared": true, @@ -8274,7 +8274,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "99% Completed commands duration", + "title": "gRPC 99% Completed commands duration", "tooltip": { "msResolution": false, "shared": true, @@ -8326,6 +8326,216 @@ "y": 27 }, "id": 124, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": null, + "description": "The rate of completing each kind of HTTP requests", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 119 + }, + "id": 1461, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(pd_service_audit_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[1m])) by (service,ip)", + "intervalFactor": 2, + "legendFormat": "{{service}}-{{ip}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Completed commands rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time consumed of completing each kind of HTTP request in .99", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 119 + }, + "id": 1462, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])) by (service, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{service}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP 99% Completed commands duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": null, + "title": "HTTP", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 125, "panels": [ { "aliasColors": {}, @@ -9414,9 +9624,9 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 29 }, - "id": 125, + "id": 126, "panels": [ { "aliasColors": {}, @@ -9870,9 +10080,9 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 30 }, - "id": 126, + "id": 127, "panels": [ { "aliasColors": {}, @@ -11059,7 +11269,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 31 }, "id": 1420, "panels": [ @@ -11531,9 +11741,9 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 32 }, - "id": 127, + "id": 128, "panels": [ { "aliasColors": {}, diff --git a/pkg/audit/audit.go b/pkg/audit/audit.go index a1f0f131126d..5063225cf630 100644 --- a/pkg/audit/audit.go +++ b/pkg/audit/audit.go @@ -98,7 +98,7 @@ func (b *PrometheusHistogramBackend) ProcessHTTPRequest(req *http.Request) bool if !ok { return false } - b.histogramVec.WithLabelValues(requestInfo.ServiceLabel, "HTTP", requestInfo.Component).Observe(float64(endTime - requestInfo.StartTimeStamp)) + b.histogramVec.WithLabelValues(requestInfo.ServiceLabel, "HTTP", requestInfo.Component, requestInfo.IP).Observe(float64(endTime - requestInfo.StartTimeStamp)) return true } diff --git a/pkg/audit/audit_test.go b/pkg/audit/audit_test.go index 2b5bd7a82625..6384d5f6b29b 100644 --- a/pkg/audit/audit_test.go +++ b/pkg/audit/audit_test.go @@ -51,7 +51,7 @@ func TestPrometheusHistogramBackend(t *testing.T) { Name: "audit_handling_seconds_test", Help: "PD server service handling audit", Buckets: prometheus.DefBuckets, - }, []string{"service", "method", "component"}) + }, []string{"service", "method", "component", "ip"}) prometheus.MustRegister(serviceAuditHistogramTest) @@ -63,6 +63,7 @@ func TestPrometheusHistogramBackend(t *testing.T) { info := requestutil.GetRequestInfo(req) info.ServiceLabel = "test" info.Component = "user1" + info.IP = "localhost" req = req.WithContext(requestutil.WithRequestInfo(req.Context(), info)) re.False(backend.ProcessHTTPRequest(req)) @@ -84,8 +85,8 @@ func TestPrometheusHistogramBackend(t *testing.T) { defer resp.Body.Close() content, _ := io.ReadAll(resp.Body) output := string(content) - re.Contains(output, "pd_service_audit_handling_seconds_test_count{component=\"user1\",method=\"HTTP\",service=\"test\"} 2") - re.Contains(output, "pd_service_audit_handling_seconds_test_count{component=\"user2\",method=\"HTTP\",service=\"test\"} 1") + re.Contains(output, "pd_service_audit_handling_seconds_test_count{component=\"user1\",ip=\"localhost\",method=\"HTTP\",service=\"test\"} 2") + re.Contains(output, "pd_service_audit_handling_seconds_test_count{component=\"user2\",ip=\"localhost\",method=\"HTTP\",service=\"test\"} 1") } func TestLocalLogBackendUsingFile(t *testing.T) { diff --git a/pkg/errs/errs_test.go b/pkg/errs/errs_test.go index 591d9f899ce9..c242dd994f59 100644 --- a/pkg/errs/errs_test.go +++ b/pkg/errs/errs_test.go @@ -63,7 +63,7 @@ func newZapTestLogger(cfg *log.Config, opts ...zap.Option) verifyLogger { // TestingWriter is used to write to memory. // Used in the verify logger. writer := newTestingWriter() - lg, _, _ := log.InitLoggerWithWriteSyncer(cfg, writer, opts...) + lg, _, _ := log.InitLoggerWithWriteSyncer(cfg, writer, writer, opts...) return verifyLogger{ Logger: lg, diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index 99a6b7df54fc..8b38fa50ae6f 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -778,6 +778,10 @@ func (mc *Cluster) AddSuspectRegions(ids ...uint64) { } } +// SetHotPendingInfluenceMetrics mock method +func (mc *Cluster) SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { +} + // GetBasicCluster mock method func (mc *Cluster) GetBasicCluster() *core.BasicCluster { return mc.BasicCluster diff --git a/plugin/scheduler_example/evict_leader.go b/plugin/scheduler_example/evict_leader.go index aee45146e46d..ef5a50a68031 100644 --- a/plugin/scheduler_example/evict_leader.go +++ b/plugin/scheduler_example/evict_leader.go @@ -235,7 +235,7 @@ func (s *evictLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ( log.Debug("fail to create evict leader operator", errs.ZapError(err)) continue } - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) ops = append(ops, op) } diff --git a/server/api/diagnostic_test.go b/server/api/diagnostic_test.go index 101ab92b1478..5709edc1f8ad 100644 --- a/server/api/diagnostic_test.go +++ b/server/api/diagnostic_test.go @@ -56,6 +56,7 @@ func (suite *diagnosticTestSuite) SetupSuite() { mustBootstrapCluster(re, suite.svr) mustPutStore(re, suite.svr, 1, metapb.StoreState_Up, metapb.NodeState_Serving, nil) + mustPutStore(re, suite.svr, 2, metapb.StoreState_Up, metapb.NodeState_Serving, nil) } func (suite *diagnosticTestSuite) TearDownSuite() { @@ -99,7 +100,7 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { err = tu.CheckPostJSON(testDialClient, suite.schedulerPrifex, body, tu.StatusOK(suite.Require())) suite.NoError(err) - time.Sleep(time.Millisecond * 50) + time.Sleep(time.Millisecond * 100) result = &cluster.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) suite.NoError(err) @@ -111,7 +112,7 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { suite.NoError(err) err = tu.CheckPostJSON(testDialClient, suite.schedulerPrifex+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) suite.NoError(err) - time.Sleep(time.Millisecond * 50) + time.Sleep(time.Millisecond * 100) result = &cluster.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) suite.NoError(err) @@ -122,14 +123,14 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { suite.NoError(err) err = tu.CheckPostJSON(testDialClient, suite.schedulerPrifex+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) suite.NoError(err) - time.Sleep(time.Millisecond * 50) + time.Sleep(time.Millisecond * 100) result = &cluster.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) suite.NoError(err) suite.Equal("pending", result.Status) mustPutRegion(re, suite.svr, 1000, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) - time.Sleep(time.Millisecond * 50) + time.Sleep(time.Millisecond * 100) result = &cluster.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) suite.NoError(err) diff --git a/server/api/router.go b/server/api/router.go index 972794798315..e98bd86f4b3d 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -103,8 +103,9 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { } } + // localLog should be used in modifying the configuration or admin operations. localLog := audit.LocalLogLabel - // Please don't use PrometheusHistogram in the hot path. + // prometheus will be used in all API. prometheus := audit.PrometheusHistogram setRateLimitAllowList := func() createRouteOption { @@ -126,105 +127,105 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { escapeRouter := clusterRouter.NewRoute().Subrouter().UseEncodedPath() operatorHandler := newOperatorHandler(handler, rd) - registerFunc(apiRouter, "/operators", operatorHandler.GetOperators, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/operators", operatorHandler.CreateOperator, setMethods(http.MethodPost), setAuditBackend(prometheus)) - registerFunc(apiRouter, "/operators/records", operatorHandler.GetOperatorRecords, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/operators/{region_id}", operatorHandler.GetOperatorsByRegion, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/operators/{region_id}", operatorHandler.DeleteOperatorByRegion, setMethods(http.MethodDelete)) + registerFunc(apiRouter, "/operators", operatorHandler.GetOperators, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/operators", operatorHandler.CreateOperator, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/operators/records", operatorHandler.GetOperatorRecords, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/operators/{region_id}", operatorHandler.GetOperatorsByRegion, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/operators/{region_id}", operatorHandler.DeleteOperatorByRegion, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) checkerHandler := newCheckerHandler(svr, rd) - registerFunc(apiRouter, "/checker/{name}", checkerHandler.PauseOrResumeChecker, setMethods(http.MethodPost)) - registerFunc(apiRouter, "/checker/{name}", checkerHandler.GetCheckerStatus, setMethods(http.MethodGet)) + registerFunc(apiRouter, "/checker/{name}", checkerHandler.PauseOrResumeChecker, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/checker/{name}", checkerHandler.GetCheckerStatus, setMethods(http.MethodGet), setAuditBackend(prometheus)) schedulerHandler := newSchedulerHandler(svr, rd) - registerFunc(apiRouter, "/schedulers", schedulerHandler.GetSchedulers, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/schedulers", schedulerHandler.CreateScheduler, setMethods(http.MethodPost)) - registerFunc(apiRouter, "/schedulers/{name}", schedulerHandler.DeleteScheduler, setMethods(http.MethodDelete)) - registerFunc(apiRouter, "/schedulers/{name}", schedulerHandler.PauseOrResumeScheduler, setMethods(http.MethodPost)) + registerFunc(apiRouter, "/schedulers", schedulerHandler.GetSchedulers, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/schedulers", schedulerHandler.CreateScheduler, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/schedulers/{name}", schedulerHandler.DeleteScheduler, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/schedulers/{name}", schedulerHandler.PauseOrResumeScheduler, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) diagnosticHandler := newDiagnosticHandler(svr, rd) - registerFunc(clusterRouter, "/schedulers/diagnostic/{name}", diagnosticHandler.GetDiagnosticResult, setMethods(http.MethodGet)) + registerFunc(clusterRouter, "/schedulers/diagnostic/{name}", diagnosticHandler.GetDiagnosticResult, setMethods(http.MethodGet), setAuditBackend(prometheus)) schedulerConfigHandler := newSchedulerConfigHandler(svr, rd) - registerPrefix(apiRouter, "/scheduler-config", schedulerConfigHandler.GetSchedulerConfig) + registerPrefix(apiRouter, "/scheduler-config", schedulerConfigHandler.GetSchedulerConfig, setAuditBackend(prometheus)) clusterHandler := newClusterHandler(svr, rd) - registerFunc(apiRouter, "/cluster", clusterHandler.GetCluster, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/cluster/status", clusterHandler.GetClusterStatus) + registerFunc(apiRouter, "/cluster", clusterHandler.GetCluster, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/cluster/status", clusterHandler.GetClusterStatus, setAuditBackend(prometheus)) confHandler := newConfHandler(svr, rd) - registerFunc(apiRouter, "/config", confHandler.GetConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config", confHandler.SetConfig, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/config/default", confHandler.GetDefaultConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config/schedule", confHandler.GetScheduleConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config/schedule", confHandler.SetScheduleConfig, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/config/pd-server", confHandler.GetPDServerConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config/replicate", confHandler.GetReplicationConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config/replicate", confHandler.SetReplicationConfig, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/config/label-property", confHandler.GetLabelPropertyConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config/label-property", confHandler.SetLabelPropertyConfig, setMethods(http.MethodPost)) - registerFunc(apiRouter, "/config/cluster-version", confHandler.GetClusterVersion, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config/cluster-version", confHandler.SetClusterVersion, setMethods(http.MethodPost)) - registerFunc(apiRouter, "/config/replication-mode", confHandler.GetReplicationModeConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/config/replication-mode", confHandler.SetReplicationModeConfig, setMethods(http.MethodPost)) + registerFunc(apiRouter, "/config", confHandler.GetConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config", confHandler.SetConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/config/default", confHandler.GetDefaultConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config/schedule", confHandler.GetScheduleConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config/schedule", confHandler.SetScheduleConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/config/pd-server", confHandler.GetPDServerConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config/replicate", confHandler.GetReplicationConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config/replicate", confHandler.SetReplicationConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/config/label-property", confHandler.GetLabelPropertyConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config/label-property", confHandler.SetLabelPropertyConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/config/cluster-version", confHandler.GetClusterVersion, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config/cluster-version", confHandler.SetClusterVersion, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/config/replication-mode", confHandler.GetReplicationModeConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/config/replication-mode", confHandler.SetReplicationModeConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) rulesHandler := newRulesHandler(svr, rd) - registerFunc(clusterRouter, "/config/rules", rulesHandler.GetAllRules, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/rules", rulesHandler.SetAllRules, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/config/rules/batch", rulesHandler.BatchRules, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/config/rules/group/{group}", rulesHandler.GetRuleByGroup, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/rules/region/{region}", rulesHandler.GetRulesByRegion, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/rules/region/{region}/detail", rulesHandler.CheckRegionPlacementRule, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/rules/key/{key}", rulesHandler.GetRulesByKey, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/rule/{group}/{id}", rulesHandler.GetRuleByGroupAndID, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/rule", rulesHandler.SetRule, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/config/rule/{group}/{id}", rulesHandler.DeleteRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog)) - - registerFunc(clusterRouter, "/config/rule_group/{id}", rulesHandler.GetGroupConfig, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/rule_group", rulesHandler.SetGroupConfig, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/config/rule_group/{id}", rulesHandler.DeleteGroupConfig, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/config/rule_groups", rulesHandler.GetAllGroupConfigs, setMethods(http.MethodGet)) - - registerFunc(clusterRouter, "/config/placement-rule", rulesHandler.GetPlacementRules, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/placement-rule", rulesHandler.SetPlacementRules, setMethods(http.MethodPost), setAuditBackend(localLog)) + registerFunc(clusterRouter, "/config/rules", rulesHandler.GetAllRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/rules", rulesHandler.SetAllRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/config/rules/batch", rulesHandler.BatchRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/config/rules/group/{group}", rulesHandler.GetRuleByGroup, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/rules/region/{region}", rulesHandler.GetRulesByRegion, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/rules/region/{region}/detail", rulesHandler.CheckRegionPlacementRule, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/rules/key/{key}", rulesHandler.GetRulesByKey, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/rule/{group}/{id}", rulesHandler.GetRuleByGroupAndID, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/rule", rulesHandler.SetRule, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/config/rule/{group}/{id}", rulesHandler.DeleteRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + + registerFunc(clusterRouter, "/config/rule_group/{id}", rulesHandler.GetGroupConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/rule_group", rulesHandler.SetGroupConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/config/rule_group/{id}", rulesHandler.DeleteGroupConfig, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/config/rule_groups", rulesHandler.GetAllGroupConfigs, setMethods(http.MethodGet), setAuditBackend(prometheus)) + + registerFunc(clusterRouter, "/config/placement-rule", rulesHandler.GetPlacementRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/placement-rule", rulesHandler.SetPlacementRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) // {group} can be a regular expression, we should enable path encode to // support special characters. - registerFunc(clusterRouter, "/config/placement-rule/{group}", rulesHandler.GetPlacementRuleByGroup, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/placement-rule/{group}", rulesHandler.SetPlacementRuleByGroup, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(escapeRouter, "/config/placement-rule/{group}", rulesHandler.DeletePlacementRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog)) + registerFunc(clusterRouter, "/config/placement-rule/{group}", rulesHandler.GetPlacementRuleByGroup, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/placement-rule/{group}", rulesHandler.SetPlacementRuleByGroup, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(escapeRouter, "/config/placement-rule/{group}", rulesHandler.DeletePlacementRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) regionLabelHandler := newRegionLabelHandler(svr, rd) - registerFunc(clusterRouter, "/config/region-label/rules", regionLabelHandler.GetAllRegionLabelRules, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/config/region-label/rules/ids", regionLabelHandler.GetRegionLabelRulesByIDs, setMethods(http.MethodGet)) + registerFunc(clusterRouter, "/config/region-label/rules", regionLabelHandler.GetAllRegionLabelRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/config/region-label/rules/ids", regionLabelHandler.GetRegionLabelRulesByIDs, setMethods(http.MethodGet), setAuditBackend(prometheus)) // {id} can be a string with special characters, we should enable path encode to support it. - registerFunc(escapeRouter, "/config/region-label/rule/{id}", regionLabelHandler.GetRegionLabelRuleByID, setMethods(http.MethodGet)) - registerFunc(escapeRouter, "/config/region-label/rule/{id}", regionLabelHandler.DeleteRegionLabelRule, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/config/region-label/rule", regionLabelHandler.SetRegionLabelRule, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/config/region-label/rules", regionLabelHandler.PatchRegionLabelRules, setMethods(http.MethodPatch), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/region/id/{id}/label/{key}", regionLabelHandler.GetRegionLabelByKey, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/region/id/{id}/labels", regionLabelHandler.GetRegionLabels, setMethods(http.MethodGet)) + registerFunc(escapeRouter, "/config/region-label/rule/{id}", regionLabelHandler.GetRegionLabelRuleByID, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(escapeRouter, "/config/region-label/rule/{id}", regionLabelHandler.DeleteRegionLabelRule, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/config/region-label/rule", regionLabelHandler.SetRegionLabelRule, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/config/region-label/rules", regionLabelHandler.PatchRegionLabelRules, setMethods(http.MethodPatch), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/region/id/{id}/label/{key}", regionLabelHandler.GetRegionLabelByKey, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/region/id/{id}/labels", regionLabelHandler.GetRegionLabels, setMethods(http.MethodGet), setAuditBackend(prometheus)) storeHandler := newStoreHandler(handler, rd) - registerFunc(clusterRouter, "/store/{id}", storeHandler.GetStore, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/store/{id}", storeHandler.DeleteStore, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/store/{id}/state", storeHandler.SetStoreState, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/store/{id}/label", storeHandler.SetStoreLabel, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/store/{id}/label", storeHandler.DeleteStoreLabel, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/store/{id}/weight", storeHandler.SetStoreWeight, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/store/{id}/limit", storeHandler.SetStoreLimit, setMethods(http.MethodPost), setAuditBackend(localLog)) + registerFunc(clusterRouter, "/store/{id}", storeHandler.GetStore, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/store/{id}", storeHandler.DeleteStore, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/store/{id}/state", storeHandler.SetStoreState, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/store/{id}/label", storeHandler.SetStoreLabel, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/store/{id}/label", storeHandler.DeleteStoreLabel, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/store/{id}/weight", storeHandler.SetStoreWeight, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/store/{id}/limit", storeHandler.SetStoreLimit, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) storesHandler := newStoresHandler(handler, rd) - registerFunc(clusterRouter, "/stores", storesHandler.GetStores, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/stores/remove-tombstone", storesHandler.RemoveTombStone, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/stores/limit", storesHandler.GetAllStoresLimit, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/stores/limit", storesHandler.SetAllStoresLimit, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/stores/limit/scene", storesHandler.SetStoreLimitScene, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/stores/limit/scene", storesHandler.GetStoreLimitScene, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/stores/progress", storesHandler.GetStoresProgress, setMethods(http.MethodGet)) + registerFunc(clusterRouter, "/stores", storesHandler.GetStores, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/stores/remove-tombstone", storesHandler.RemoveTombStone, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/stores/limit", storesHandler.GetAllStoresLimit, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/stores/limit", storesHandler.SetAllStoresLimit, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/stores/limit/scene", storesHandler.SetStoreLimitScene, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/stores/limit/scene", storesHandler.GetStoreLimitScene, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/stores/progress", storesHandler.GetStoresProgress, setMethods(http.MethodGet), setAuditBackend(prometheus)) labelsHandler := newLabelsHandler(svr, rd) - registerFunc(clusterRouter, "/labels", labelsHandler.GetLabels, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/labels/stores", labelsHandler.GetStoresByLabel, setMethods(http.MethodGet)) + registerFunc(clusterRouter, "/labels", labelsHandler.GetLabels, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/labels/stores", labelsHandler.GetStoresByLabel, setMethods(http.MethodGet), setAuditBackend(prometheus)) hotStatusHandler := newHotStatusHandler(handler, rd) registerFunc(apiRouter, "/hotspot/regions/write", hotStatusHandler.GetHotWriteRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) @@ -243,89 +244,89 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { regionsHandler := newRegionsHandler(svr, rd) registerFunc(clusterRouter, "/regions/key", regionsHandler.ScanRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/count", regionsHandler.GetRegionCount, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/regions/store/{id}", regionsHandler.GetStoreRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/writeflow", regionsHandler.GetTopWriteFlowRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/readflow", regionsHandler.GetTopReadFlowRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/confver", regionsHandler.GetTopConfVerRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/version", regionsHandler.GetTopVersionRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/size", regionsHandler.GetTopSizeRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/keys", regionsHandler.GetTopKeysRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/miss-peer", regionsHandler.GetMissPeerRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/extra-peer", regionsHandler.GetExtraPeerRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/pending-peer", regionsHandler.GetPendingPeerRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/down-peer", regionsHandler.GetDownPeerRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/learner-peer", regionsHandler.GetLearnerPeerRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/empty-region", regionsHandler.GetEmptyRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/offline-peer", regionsHandler.GetOfflinePeerRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/oversized-region", regionsHandler.GetOverSizedRegions, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/undersized-region", regionsHandler.GetUndersizedRegions, setMethods(http.MethodGet)) - - registerFunc(clusterRouter, "/regions/check/hist-size", regionsHandler.GetSizeHistogram, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/check/hist-keys", regionsHandler.GetKeysHistogram, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/sibling/{id}", regionsHandler.GetRegionSiblings, setMethods(http.MethodGet)) + registerFunc(clusterRouter, "/regions/store/{id}", regionsHandler.GetStoreRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/writeflow", regionsHandler.GetTopWriteFlowRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/readflow", regionsHandler.GetTopReadFlowRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/confver", regionsHandler.GetTopConfVerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/version", regionsHandler.GetTopVersionRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/size", regionsHandler.GetTopSizeRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/keys", regionsHandler.GetTopKeysRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/miss-peer", regionsHandler.GetMissPeerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/extra-peer", regionsHandler.GetExtraPeerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/pending-peer", regionsHandler.GetPendingPeerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/down-peer", regionsHandler.GetDownPeerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/learner-peer", regionsHandler.GetLearnerPeerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/empty-region", regionsHandler.GetEmptyRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/offline-peer", regionsHandler.GetOfflinePeerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/oversized-region", regionsHandler.GetOverSizedRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/undersized-region", regionsHandler.GetUndersizedRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + + registerFunc(clusterRouter, "/regions/check/hist-size", regionsHandler.GetSizeHistogram, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/check/hist-keys", regionsHandler.GetKeysHistogram, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/sibling/{id}", regionsHandler.GetRegionSiblings, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/accelerate-schedule", regionsHandler.AccelerateRegionsScheduleInRange, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(clusterRouter, "/regions/scatter", regionsHandler.ScatterRegions, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(clusterRouter, "/regions/split", regionsHandler.SplitRegions, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/regions/range-holes", regionsHandler.GetRangeHoles, setMethods(http.MethodGet)) - registerFunc(clusterRouter, "/regions/replicated", regionsHandler.CheckRegionsReplicated, setMethods(http.MethodGet), setQueries("startKey", "{startKey}", "endKey", "{endKey}")) + registerFunc(clusterRouter, "/regions/split", regionsHandler.SplitRegions, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/regions/range-holes", regionsHandler.GetRangeHoles, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/replicated", regionsHandler.CheckRegionsReplicated, setMethods(http.MethodGet), setQueries("startKey", "{startKey}", "endKey", "{endKey}"), setAuditBackend(prometheus)) - registerFunc(apiRouter, "/version", newVersionHandler(rd).GetVersion, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/status", newStatusHandler(svr, rd).GetPDStatus, setMethods(http.MethodGet)) + registerFunc(apiRouter, "/version", newVersionHandler(rd).GetVersion, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/status", newStatusHandler(svr, rd).GetPDStatus, setMethods(http.MethodGet), setAuditBackend(prometheus)) memberHandler := newMemberHandler(svr, rd) - registerFunc(apiRouter, "/members", memberHandler.GetMembers, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/members/name/{name}", memberHandler.DeleteMemberByName, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(apiRouter, "/members/id/{id}", memberHandler.DeleteMemberByID, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(apiRouter, "/members/name/{name}", memberHandler.SetMemberPropertyByName, setMethods(http.MethodPost), setAuditBackend(localLog)) + registerFunc(apiRouter, "/members", memberHandler.GetMembers, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/members/name/{name}", memberHandler.DeleteMemberByName, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/members/id/{id}", memberHandler.DeleteMemberByID, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/members/name/{name}", memberHandler.SetMemberPropertyByName, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) leaderHandler := newLeaderHandler(svr, rd) - registerFunc(apiRouter, "/leader", leaderHandler.GetLeader, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/leader/resign", leaderHandler.ResignLeader, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/leader/transfer/{next_leader}", leaderHandler.TransferLeader, setMethods(http.MethodPost), setAuditBackend(localLog)) + registerFunc(apiRouter, "/leader", leaderHandler.GetLeader, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/leader/resign", leaderHandler.ResignLeader, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/leader/transfer/{next_leader}", leaderHandler.TransferLeader, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) statsHandler := newStatsHandler(svr, rd) - registerFunc(clusterRouter, "/stats/region", statsHandler.GetRegionStatus, setMethods(http.MethodGet)) + registerFunc(clusterRouter, "/stats/region", statsHandler.GetRegionStatus, setMethods(http.MethodGet), setAuditBackend(prometheus)) trendHandler := newTrendHandler(svr, rd) registerFunc(apiRouter, "/trend", trendHandler.GetTrend, setMethods(http.MethodGet), setAuditBackend(prometheus)) adminHandler := newAdminHandler(svr, rd) - registerFunc(clusterRouter, "/admin/cache/region/{id}", adminHandler.DeleteRegionCache, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(clusterRouter, "/admin/cache/regions", adminHandler.DeleteAllRegionCache, setMethods(http.MethodDelete), setAuditBackend(localLog)) + registerFunc(clusterRouter, "/admin/cache/region/{id}", adminHandler.DeleteRegionCache, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/admin/cache/regions", adminHandler.DeleteAllRegionCache, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) // br ebs restore phase 1 will reset ts, but at that time the cluster hasn't bootstrapped, so cannot use clusterRouter - registerFunc(apiRouter, "/admin/reset-ts", adminHandler.ResetTS, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.IsSnapshotRecovering, setMethods(http.MethodGet), setAuditBackend(localLog)) - registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.MarkSnapshotRecovering, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.UnmarkSnapshotRecovering, setMethods(http.MethodDelete), setAuditBackend(localLog)) - registerFunc(apiRouter, "/admin/base-alloc-id", adminHandler.RecoverAllocID, setMethods(http.MethodPost), setAuditBackend(localLog)) + registerFunc(apiRouter, "/admin/reset-ts", adminHandler.ResetTS, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.IsSnapshotRecovering, setMethods(http.MethodGet), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.MarkSnapshotRecovering, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.UnmarkSnapshotRecovering, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/base-alloc-id", adminHandler.RecoverAllocID, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) serviceMiddlewareHandler := newServiceMiddlewareHandler(svr, rd) - registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.GetServiceMiddlewareConfig, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.SetServiceMiddlewareConfig, setMethods(http.MethodPost), setAuditBackend(localLog)) - registerFunc(apiRouter, "/service-middleware/config/rate-limit", serviceMiddlewareHandler.SetRatelimitConfig, setMethods(http.MethodPost), setAuditBackend(localLog), setRateLimitAllowList()) + registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.GetServiceMiddlewareConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.SetServiceMiddlewareConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/service-middleware/config/rate-limit", serviceMiddlewareHandler.SetRatelimitConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus), setRateLimitAllowList()) logHandler := newLogHandler(svr, rd) - registerFunc(apiRouter, "/admin/log", logHandler.SetLogLevel, setMethods(http.MethodPost), setAuditBackend(localLog)) + registerFunc(apiRouter, "/admin/log", logHandler.SetLogLevel, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) replicationModeHandler := newReplicationModeHandler(svr, rd) - registerFunc(clusterRouter, "/replication_mode/status", replicationModeHandler.GetReplicationModeStatus) + registerFunc(clusterRouter, "/replication_mode/status", replicationModeHandler.GetReplicationModeStatus, setAuditBackend(prometheus)) pluginHandler := newPluginHandler(handler, rd) - registerFunc(apiRouter, "/plugin", pluginHandler.LoadPlugin, setMethods(http.MethodPost)) - registerFunc(apiRouter, "/plugin", pluginHandler.UnloadPlugin, setMethods(http.MethodDelete)) + registerFunc(apiRouter, "/plugin", pluginHandler.LoadPlugin, setMethods(http.MethodPost), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/plugin", pluginHandler.UnloadPlugin, setMethods(http.MethodDelete), setAuditBackend(prometheus)) healthHandler := newHealthHandler(svr, rd) - registerFunc(apiRouter, "/health", healthHandler.GetHealthStatus, setMethods(http.MethodGet)) - registerFunc(apiRouter, "/ping", healthHandler.Ping, setMethods(http.MethodGet)) + registerFunc(apiRouter, "/health", healthHandler.GetHealthStatus, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/ping", healthHandler.Ping, setMethods(http.MethodGet), setAuditBackend(prometheus)) // metric query use to query metric data, the protocol is compatible with prometheus. - registerFunc(apiRouter, "/metric/query", newQueryMetric(svr).QueryMetric, setMethods(http.MethodGet, http.MethodPost)) - registerFunc(apiRouter, "/metric/query_range", newQueryMetric(svr).QueryMetric, setMethods(http.MethodGet, http.MethodPost)) + registerFunc(apiRouter, "/metric/query", newQueryMetric(svr).QueryMetric, setMethods(http.MethodGet, http.MethodPost), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/metric/query_range", newQueryMetric(svr).QueryMetric, setMethods(http.MethodGet, http.MethodPost), setAuditBackend(prometheus)) // tso API tsoHandler := newTSOHandler(svr, rd) - registerFunc(apiRouter, "/tso/allocator/transfer/{name}", tsoHandler.TransferLocalTSOAllocator, setMethods(http.MethodPost), setAuditBackend(localLog)) + registerFunc(apiRouter, "/tso/allocator/transfer/{name}", tsoHandler.TransferLocalTSOAllocator, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) pprofHandler := newPprofHandler(svr, rd) // profile API @@ -342,19 +343,19 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { // service GC safepoint API serviceGCSafepointHandler := newServiceGCSafepointHandler(svr, rd) - registerFunc(apiRouter, "/gc/safepoint", serviceGCSafepointHandler.GetGCSafePoint, setMethods(http.MethodGet), setAuditBackend(localLog)) - registerFunc(apiRouter, "/gc/safepoint/{service_id}", serviceGCSafepointHandler.DeleteGCSafePoint, setMethods(http.MethodDelete), setAuditBackend(localLog)) + registerFunc(apiRouter, "/gc/safepoint", serviceGCSafepointHandler.GetGCSafePoint, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/gc/safepoint/{service_id}", serviceGCSafepointHandler.DeleteGCSafePoint, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) // min resolved ts API minResolvedTSHandler := newMinResolvedTSHandler(svr, rd) - registerFunc(clusterRouter, "/min-resolved-ts", minResolvedTSHandler.GetMinResolvedTS, setMethods(http.MethodGet)) + registerFunc(clusterRouter, "/min-resolved-ts", minResolvedTSHandler.GetMinResolvedTS, setMethods(http.MethodGet), setAuditBackend(prometheus)) // unsafe admin operation API unsafeOperationHandler := newUnsafeOperationHandler(svr, rd) registerFunc(clusterRouter, "/admin/unsafe/remove-failed-stores", - unsafeOperationHandler.RemoveFailedStores, setMethods(http.MethodPost)) + unsafeOperationHandler.RemoveFailedStores, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(clusterRouter, "/admin/unsafe/remove-failed-stores/show", - unsafeOperationHandler.GetFailedStoresRemovalStatus, setMethods(http.MethodGet)) + unsafeOperationHandler.GetFailedStoresRemovalStatus, setMethods(http.MethodGet), setAuditBackend(prometheus)) // API to set or unset failpoints failpoint.Inject("enableFailpointAPI", func() { diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index 9b2ec7d87461..0c037c2cc665 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -59,7 +59,7 @@ func (suite *auditMiddlewareTestSuite) TestConfigAuditSwitch() { sc := &config.ServiceMiddlewareConfig{} re := suite.Require() suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.False(sc.EnableAudit) + suite.True(sc.EnableAudit) ms := map[string]interface{}{ "enable-audit": "true", diff --git a/server/api/trend_test.go b/server/api/trend_test.go index 9655faf0b7a2..cdf27206e8a2 100644 --- a/server/api/trend_test.go +++ b/server/api/trend_test.go @@ -39,9 +39,9 @@ func TestTrend(t *testing.T) { } // Create 3 regions, all peers on store1 and store2, and the leaders are all on store1. - region4 := newRegionInfo(4, "", "a", 2, 2, []uint64{1, 2}, nil, 1) - region5 := newRegionInfo(5, "a", "b", 2, 2, []uint64{1, 2}, nil, 1) - region6 := newRegionInfo(6, "b", "", 2, 2, []uint64{1, 2}, nil, 1) + region4 := newRegionInfo(4, "", "a", 2, 2, []uint64{1, 2}, nil, nil, 1) + region5 := newRegionInfo(5, "a", "b", 2, 2, []uint64{1, 2}, nil, []uint64{2}, 1) + region6 := newRegionInfo(6, "b", "", 2, 2, []uint64{1, 2}, nil, nil, 1) mustRegionHeartbeat(re, svr, region4) mustRegionHeartbeat(re, svr, region5) mustRegionHeartbeat(re, svr, region6) @@ -57,6 +57,8 @@ func TestTrend(t *testing.T) { op, err := svr.GetHandler().GetOperator(5) re.NoError(err) re.NotNil(op) + re.True(op.Step(0).(operator.AddLearner).IsWitness) + newPeerID := op.Step(0).(operator.AddLearner).PeerID region5 = region5.Clone(core.WithAddPeer(&metapb.Peer{Id: newPeerID, StoreId: 3, Role: metapb.PeerRole_Learner}), core.WithIncConfVer()) mustRegionHeartbeat(re, svr, region5) @@ -97,20 +99,34 @@ func TestTrend(t *testing.T) { } } -func newRegionInfo(id uint64, startKey, endKey string, confVer, ver uint64, voters []uint64, learners []uint64, leaderStore uint64) *core.RegionInfo { +func newRegionInfo(id uint64, startKey, endKey string, confVer, ver uint64, voters []uint64, learners []uint64, witnesses []uint64, leaderStore uint64) *core.RegionInfo { var ( peers = make([]*metapb.Peer, 0, len(voters)+len(learners)) leader *metapb.Peer ) for _, id := range voters { - p := &metapb.Peer{Id: 10 + id, StoreId: id} + witness := false + for _, wid := range witnesses { + if id == wid { + witness = true + break + } + } + p := &metapb.Peer{Id: 10 + id, StoreId: id, IsWitness: witness} if id == leaderStore { leader = p } peers = append(peers, p) } for _, id := range learners { - p := &metapb.Peer{Id: 10 + id, StoreId: id, Role: metapb.PeerRole_Learner} + witness := false + for _, wid := range witnesses { + if id == wid { + witness = true + break + } + } + p := &metapb.Peer{Id: 10 + id, StoreId: id, Role: metapb.PeerRole_Learner, IsWitness: witness} peers = append(peers, p) } return core.NewRegionInfo( diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 75ae2a78ebc9..a29b35fac9b7 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1797,6 +1797,11 @@ func (c *RaftCluster) deleteStoreLocked(store *core.StoreInfo) error { return nil } +// SetHotPendingInfluenceMetrics sets pending influence in hot scheduler. +func (c *RaftCluster) SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { + hotPendingSum.WithLabelValues(storeLabel, rwTy, dim).Set(load) +} + func (c *RaftCluster) collectMetrics() { statsMap := statistics.NewStoreStatisticsMap(c.opt, c.storeConfigManager.GetStoreConfig()) stores := c.GetStores() diff --git a/server/cluster/coordinator.go b/server/cluster/coordinator.go index a05eaf7d99e7..581e76557e3c 100644 --- a/server/cluster/coordinator.go +++ b/server/cluster/coordinator.go @@ -539,8 +539,6 @@ func (c *coordinator) collectHotSpotMetrics() { collectHotMetrics(c.cluster, stores, statistics.Write) // Collects hot read region metrics. collectHotMetrics(c.cluster, stores, statistics.Read) - // Collects pending influence. - collectPendingInfluence(stores) } func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ statistics.RWType) { @@ -563,8 +561,8 @@ func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ stati storeAddress := s.GetAddress() storeID := s.GetID() storeLabel := strconv.FormatUint(storeID, 10) - stat, ok := status.AsLeader[storeID] - if ok { + stat, hasHotLeader := status.AsLeader[storeID] + if hasHotLeader { hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "total_"+kind+"_bytes_as_leader").Set(stat.TotalBytesRate) hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "total_"+kind+"_keys_as_leader").Set(stat.TotalKeysRate) hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "total_"+kind+"_query_as_leader").Set(stat.TotalQueryRate) @@ -576,8 +574,8 @@ func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ stati hotSpotStatusGauge.DeleteLabelValues(storeAddress, storeLabel, "hot_"+kind+"_region_as_leader") } - stat, ok = status.AsPeer[storeID] - if ok { + stat, hasHotPeer := status.AsPeer[storeID] + if hasHotPeer { hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "total_"+kind+"_bytes_as_peer").Set(stat.TotalBytesRate) hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "total_"+kind+"_keys_as_peer").Set(stat.TotalKeysRate) hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "total_"+kind+"_query_as_peer").Set(stat.TotalQueryRate) @@ -588,29 +586,18 @@ func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ stati hotSpotStatusGauge.DeleteLabelValues(storeAddress, storeLabel, "total_"+kind+"_query_as_peer") hotSpotStatusGauge.DeleteLabelValues(storeAddress, storeLabel, "hot_"+kind+"_region_as_peer") } - } -} -func collectPendingInfluence(stores []*core.StoreInfo) { - pendings := statistics.GetPendingInfluence(stores) - for _, s := range stores { - storeAddress := s.GetAddress() - storeID := s.GetID() - storeLabel := strconv.FormatUint(storeID, 10) - if infl := pendings[storeID]; infl != nil { - hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "read_pending_influence_byte_rate").Set(infl.Loads[statistics.RegionReadBytes]) - hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "read_pending_influence_key_rate").Set(infl.Loads[statistics.RegionReadKeys]) - hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "read_pending_influence_query_rate").Set(infl.Loads[statistics.RegionReadQueryNum]) - hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "write_pending_influence_byte_rate").Set(infl.Loads[statistics.RegionWriteBytes]) - hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "write_pending_influence_key_rate").Set(infl.Loads[statistics.RegionWriteKeys]) - hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "write_pending_influence_query_rate").Set(infl.Loads[statistics.RegionWriteQueryNum]) - hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "pending_influence_count").Set(infl.Count) + if !hasHotLeader && !hasHotPeer { + statistics.ForeachRegionStats(func(rwTy statistics.RWType, dim int, _ statistics.RegionStatKind) { + hotPendingSum.DeleteLabelValues(storeLabel, rwTy.String(), statistics.DimToString(dim)) + }) } } } func (c *coordinator) resetHotSpotMetrics() { hotSpotStatusGauge.Reset() + hotPendingSum.Reset() } func (c *coordinator) shouldRun() bool { diff --git a/server/cluster/coordinator_test.go b/server/cluster/coordinator_test.go index 0f2c61497677..84c43b7e60bf 100644 --- a/server/cluster/coordinator_test.go +++ b/server/cluster/coordinator_test.go @@ -1016,7 +1016,7 @@ func TestOperatorCount(t *testing.T) { re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) // 1:region 2:leader re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion) - op2.SetPriorityLevel(core.HighPriority) + op2.SetPriorityLevel(core.High) oc.AddWaitingOperator(op2) re.Equal(uint64(2), oc.OperatorCount(operator.OpRegion)) // 1:region 2:region re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) @@ -1099,7 +1099,7 @@ func TestStoreOverloadedWithReplace(t *testing.T) { op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 1}) re.True(oc.AddOperator(op1)) op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 2}) - op2.SetPriorityLevel(core.HighPriority) + op2.SetPriorityLevel(core.High) re.True(oc.AddOperator(op2)) op3 := newTestOperator(1, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 3}) re.False(oc.AddOperator(op3)) @@ -1211,7 +1211,7 @@ func TestController(t *testing.T) { // add a PriorityKind operator will remove old operator { op3 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpHotRegion) - op3.SetPriorityLevel(core.HighPriority) + op3.SetPriorityLevel(core.High) re.Equal(1, oc.AddWaitingOperator(op11)) re.False(sc.AllowSchedule(false)) re.Equal(1, oc.AddWaitingOperator(op3)) @@ -1225,7 +1225,7 @@ func TestController(t *testing.T) { re.Equal(1, oc.AddWaitingOperator(op2)) re.False(sc.AllowSchedule(false)) op4 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpAdmin) - op4.SetPriorityLevel(core.HighPriority) + op4.SetPriorityLevel(core.High) re.Equal(1, oc.AddWaitingOperator(op4)) re.True(sc.AllowSchedule(false)) re.True(oc.RemoveOperator(op4)) diff --git a/server/cluster/metrics.go b/server/cluster/metrics.go index 8afb441d65ff..8ebafcab46f4 100644 --- a/server/cluster/metrics.go +++ b/server/cluster/metrics.go @@ -57,6 +57,14 @@ var ( Help: "Status of the hotspot.", }, []string{"address", "store", "type"}) + hotPendingSum = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_pending_sum", + Help: "Pending influence sum of store in hot region scheduler.", + }, []string{"store", "rw", "dim"}) + patrolCheckRegionsGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "pd", diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index 38f51fce3fdf..3d748de1c65b 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -17,7 +17,7 @@ package config import "github.com/tikv/pd/pkg/ratelimit" const ( - defaultEnableAuditMiddleware = false + defaultEnableAuditMiddleware = true defaultEnableRateLimitMiddleware = false ) diff --git a/server/core/kind.go b/server/core/kind.go index 5af3d236a4c8..4c99bd605a8d 100644 --- a/server/core/kind.go +++ b/server/core/kind.go @@ -19,9 +19,10 @@ type PriorityLevel int // Built-in priority level const ( - LowPriority PriorityLevel = iota - NormalPriority - HighPriority + Low PriorityLevel = iota + Medium + High + Urgent ) // ScheduleKind distinguishes resources and schedule policy. diff --git a/server/core/peer.go b/server/core/peer.go index 9d324b29374f..81a154b45726 100644 --- a/server/core/peer.go +++ b/server/core/peer.go @@ -36,9 +36,6 @@ func IsVoter(peer *metapb.Peer) bool { // IsVoterOrIncomingVoter judges whether peer role will become Voter. // The peer is not nil and the role is equal to IncomingVoter or Voter. func IsVoterOrIncomingVoter(peer *metapb.Peer) bool { - if peer == nil { - return false - } switch peer.GetRole() { case metapb.PeerRole_IncomingVoter, metapb.PeerRole_Voter: return true @@ -49,9 +46,6 @@ func IsVoterOrIncomingVoter(peer *metapb.Peer) bool { // IsLearnerOrDemotingVoter judges whether peer role will become Learner. // The peer is not nil and the role is equal to DemotingVoter or Learner. func IsLearnerOrDemotingVoter(peer *metapb.Peer) bool { - if peer == nil { - return false - } switch peer.GetRole() { case metapb.PeerRole_DemotingVoter, metapb.PeerRole_Learner: return true diff --git a/server/handler.go b/server/handler.go index 914fe7c6cb72..bb168004807e 100644 --- a/server/handler.go +++ b/server/handler.go @@ -629,7 +629,7 @@ func (h *Handler) AddTransferPeerOperator(regionID uint64, fromStoreID, toStoreI return err } - newPeer := &metapb.Peer{StoreId: toStoreID, Role: oldPeer.GetRole()} + newPeer := &metapb.Peer{StoreId: toStoreID, Role: oldPeer.GetRole(), IsWitness: oldPeer.GetIsWitness()} op, err := operator.CreateMovePeerOperator("admin-move-peer", c, region, operator.OpAdmin, fromStoreID, newPeer) if err != nil { log.Debug("fail to create move peer operator", errs.ZapError(err)) diff --git a/server/metrics.go b/server/metrics.go index e44c04ce9e0b..4a3efaeebbcd 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -142,7 +142,7 @@ var ( Name: "audit_handling_seconds", Help: "PD server service handling audit", Buckets: prometheus.DefBuckets, - }, []string{"service", "method", "component"}) + }, []string{"service", "method", "component", "ip"}) ) func init() { diff --git a/server/schedule/checker/joint_state_checker.go b/server/schedule/checker/joint_state_checker.go index 1e300f719598..cc1edbea5ba3 100644 --- a/server/schedule/checker/joint_state_checker.go +++ b/server/schedule/checker/joint_state_checker.go @@ -55,7 +55,7 @@ func (c *JointStateChecker) Check(region *core.RegionInfo) *operator.Operator { if op.Len() > 1 { checkerCounter.WithLabelValues("joint_state_checker", "transfer-leader").Inc() } - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) } return op } diff --git a/server/schedule/checker/replica_checker.go b/server/schedule/checker/replica_checker.go index 1dd76b6da36f..fe092a36c465 100644 --- a/server/schedule/checker/replica_checker.go +++ b/server/schedule/checker/replica_checker.go @@ -72,17 +72,17 @@ func (r *ReplicaChecker) Check(region *core.RegionInfo) *operator.Operator { } if op := r.checkDownPeer(region); op != nil { checkerCounter.WithLabelValues("replica_checker", "new-operator").Inc() - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) return op } if op := r.checkOfflinePeer(region); op != nil { checkerCounter.WithLabelValues("replica_checker", "new-operator").Inc() - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) return op } if op := r.checkMakeUpReplica(region); op != nil { checkerCounter.WithLabelValues("replica_checker", "new-operator").Inc() - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) return op } if op := r.checkRemoveExtraReplica(region); op != nil { diff --git a/server/schedule/checker/rule_checker.go b/server/schedule/checker/rule_checker.go index 718ef36d5d86..8596af39412e 100644 --- a/server/schedule/checker/rule_checker.go +++ b/server/schedule/checker/rule_checker.go @@ -33,11 +33,12 @@ import ( ) var ( - errNoStoreToAdd = errors.New("no store to add peer") - errNoStoreToReplace = errors.New("no store to replace peer") - errPeerCannotBeLeader = errors.New("peer cannot be leader") - errNoNewLeader = errors.New("no new leader") - errRegionNoLeader = errors.New("region no leader") + errNoStoreToAdd = errors.New("no store to add peer") + errNoStoreToReplace = errors.New("no store to replace peer") + errPeerCannotBeLeader = errors.New("peer cannot be leader") + errPeerCannotBeWitness = errors.New("peer cannot be witness") + errNoNewLeader = errors.New("no new leader") + errRegionNoLeader = errors.New("region no leader") ) const maxPendingListLen = 100000 @@ -181,12 +182,12 @@ func (c *RuleChecker) addRulePeer(region *core.RegionInfo, rf *placement.RuleFit c.handleFilterState(region, filterByTempState) return nil, errNoStoreToAdd } - peer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole()} + peer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole(), IsWitness: rf.Rule.IsWitness} op, err := operator.CreateAddPeerOperator("add-rule-peer", c.cluster, region, peer, operator.OpReplica) if err != nil { return nil, err } - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) return op, nil } @@ -199,7 +200,7 @@ func (c *RuleChecker) replaceUnexpectRulePeer(region *core.RegionInfo, rf *place c.handleFilterState(region, filterByTempState) return nil, errNoStoreToReplace } - newPeer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole()} + newPeer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole(), IsWitness: rf.Rule.IsWitness} // pick the smallest leader store to avoid the Offline store be snapshot generator bottleneck. var newLeader *metapb.Peer if region.GetLeader().GetId() == peer.GetId() { @@ -235,7 +236,7 @@ func (c *RuleChecker) replaceUnexpectRulePeer(region *core.RegionInfo, rf *place if newLeader != nil { c.record.incOfflineLeaderCount(newLeader.GetStoreId()) } - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) return op, nil } @@ -266,6 +267,24 @@ func (c *RuleChecker) fixLooseMatchPeer(region *core.RegionInfo, fit *placement. checkerCounter.WithLabelValues("rule_checker", "demote-voter-role").Inc() return operator.CreateDemoteVoterOperator("fix-demote-voter", c.cluster, region, peer) } + if region.GetLeader().GetId() == peer.GetId() && rf.Rule.IsWitness { + return nil, errPeerCannotBeWitness + } + if !core.IsWitness(peer) && rf.Rule.IsWitness { + lv := "set-voter-witness" + if core.IsLearner(peer) { + lv = "set-learner-witness" + } + checkerCounter.WithLabelValues("rule_checker", lv).Inc() + return operator.CreateWitnessPeerOperator("fix-witness-peer", c.cluster, region, peer) + } else if core.IsWitness(peer) && !rf.Rule.IsWitness { + lv := "set-voter-non-witness" + if core.IsLearner(peer) { + lv = "set-learner-non-witness" + } + checkerCounter.WithLabelValues("rule_checker", lv).Inc() + return operator.CreateNonWitnessPeerOperator("fix-non-witness-peer", c.cluster, region, peer) + } return nil, nil } @@ -308,7 +327,7 @@ func (c *RuleChecker) fixBetterLocation(region *core.RegionInfo, rf *placement.R return nil, nil } checkerCounter.WithLabelValues("rule_checker", "move-to-better-location").Inc() - newPeer := &metapb.Peer{StoreId: newStore, Role: rf.Rule.Role.MetaPeerRole()} + newPeer := &metapb.Peer{StoreId: newStore, Role: rf.Rule.Role.MetaPeerRole(), IsWitness: rf.Rule.IsWitness} return operator.CreateMovePeerOperator("move-to-better-location", c.cluster, region, operator.OpReplica, oldStore, newPeer) } diff --git a/server/schedule/checker/rule_checker_test.go b/server/schedule/checker/rule_checker_test.go index a07a68575ab1..93691a549ec2 100644 --- a/server/schedule/checker/rule_checker_test.go +++ b/server/schedule/checker/rule_checker_test.go @@ -67,7 +67,7 @@ func (suite *ruleCheckerTestSuite) TestAddRulePeer() { op := suite.rc.Check(suite.cluster.GetRegion(1)) suite.NotNil(op) suite.Equal("add-rule-peer", op.Desc()) - suite.Equal(core.HighPriority, op.GetPriorityLevel()) + suite.Equal(core.High, op.GetPriorityLevel()) suite.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) } @@ -120,7 +120,7 @@ func (suite *ruleCheckerTestSuite) TestFixPeer() { op = suite.rc.Check(r) suite.NotNil(op) suite.Equal("replace-rule-down-peer", op.Desc()) - suite.Equal(core.HighPriority, op.GetPriorityLevel()) + suite.Equal(core.High, op.GetPriorityLevel()) var add operator.AddLearner suite.IsType(add, op.Step(0)) suite.cluster.SetStoreUp(2) @@ -128,7 +128,7 @@ func (suite *ruleCheckerTestSuite) TestFixPeer() { op = suite.rc.Check(suite.cluster.GetRegion(1)) suite.NotNil(op) suite.Equal("replace-rule-offline-peer", op.Desc()) - suite.Equal(core.HighPriority, op.GetPriorityLevel()) + suite.Equal(core.High, op.GetPriorityLevel()) suite.IsType(add, op.Step(0)) suite.cluster.SetStoreUp(2) @@ -302,6 +302,93 @@ func (suite *ruleCheckerTestSuite) TestFixLeaderRoleWithUnhealthyRegion() { suite.Nil(op) } +func (suite *ruleCheckerTestSuite) TestFixRuleWitness() { + suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "follower"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) + suite.cluster.AddLeaderRegion(1, 1, 2) + + suite.ruleManager.SetRule(&placement.Rule{ + GroupID: "pd", + ID: "r1", + Index: 100, + Override: true, + Role: placement.Voter, + Count: 1, + IsWitness: true, + LabelConstraints: []placement.LabelConstraint{ + {Key: "C", Op: "in", Values: []string{"voter"}}, + }, + }) + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.NotNil(op) + suite.Equal("add-rule-peer", op.Desc()) + suite.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) + suite.True(op.Step(0).(operator.AddLearner).IsWitness) +} + +func (suite *ruleCheckerTestSuite) TestFixRuleWitness2() { + suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) + suite.cluster.AddLeaderRegion(1, 1, 2, 3) + + suite.ruleManager.SetRule(&placement.Rule{ + GroupID: "pd", + ID: "r1", + Index: 100, + Override: true, + Role: placement.Voter, + Count: 1, + IsWitness: true, + LabelConstraints: []placement.LabelConstraint{ + {Key: "C", Op: "in", Values: []string{"voter"}}, + }, + }) + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.NotNil(op) + suite.Equal("fix-witness-peer", op.Desc()) + suite.Equal(uint64(3), op.Step(0).(operator.BecomeWitness).StoreID) +} + +func (suite *ruleCheckerTestSuite) TestFixRuleWitness3() { + suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) + suite.cluster.AddLeaderRegion(1, 1, 2, 3) + + r := suite.cluster.GetRegion(1) + // set peer3 to witness + r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) + + op := suite.rc.Check(r) + suite.NotNil(op) + suite.Equal("fix-non-witness-peer", op.Desc()) + suite.Equal(uint64(3), op.Step(0).(operator.BecomeNonWitness).StoreID) +} + +func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() { + suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) + suite.cluster.AddLeaderRegion(1, 1, 2, 3) + + suite.ruleManager.SetRule(&placement.Rule{ + GroupID: "pd", + ID: "r1", + Index: 100, + Override: true, + Role: placement.Voter, + Count: 2, + IsWitness: true, + LabelConstraints: []placement.LabelConstraint{ + {Key: "A", Op: "In", Values: []string{"leader"}}, + }, + }) + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.Nil(op) +} + func (suite *ruleCheckerTestSuite) TestBetterReplacement() { suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"}) @@ -850,7 +937,7 @@ func (suite *ruleCheckerTestSuite) TestPendingList() { op = suite.rc.Check(suite.cluster.GetRegion(1)) suite.NotNil(op) suite.Equal("add-rule-peer", op.Desc()) - suite.Equal(core.HighPriority, op.GetPriorityLevel()) + suite.Equal(core.High, op.GetPriorityLevel()) suite.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) _, exist = suite.rc.pendingList.Get(1) suite.False(exist) diff --git a/server/schedule/cluster.go b/server/schedule/cluster.go index 842b88bea98a..567ca9c96ddb 100644 --- a/server/schedule/cluster.go +++ b/server/schedule/cluster.go @@ -35,4 +35,5 @@ type Cluster interface { RemoveScheduler(name string) error AddSuspectRegions(ids ...uint64) + SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) } diff --git a/server/schedule/filter/filters.go b/server/schedule/filter/filters.go index d8b010890932..645fa99d9735 100644 --- a/server/schedule/filter/filters.go +++ b/server/schedule/filter/filters.go @@ -51,6 +51,29 @@ func SelectSourceStores(stores []*core.StoreInfo, filters []Filter, opt *config. }) } +// SelectUnavailableTargetStores selects unavailable stores that can't be selected as target store from the list. +func SelectUnavailableTargetStores(stores []*core.StoreInfo, filters []Filter, opt *config.PersistOptions, collector *plan.Collector) []*core.StoreInfo { + return filterStoresBy(stores, func(s *core.StoreInfo) bool { + targetID := strconv.FormatUint(s.GetID(), 10) + return slice.AnyOf(filters, func(i int) bool { + status := filters[i].Target(opt, s) + if !status.IsOK() { + cfilter, ok := filters[i].(comparingFilter) + sourceID := "" + if ok { + sourceID = strconv.FormatUint(cfilter.GetSourceStoreID(), 10) + } + filterCounter.WithLabelValues(filterTarget, filters[i].Scope(), filters[i].Type(), targetID, sourceID).Inc() + if collector != nil { + collector.Collect(plan.SetResourceWithStep(s, 2), plan.SetStatus(status)) + } + return true + } + return false + }) + }) +} + // SelectTargetStores selects stores that be selected as target store from the list. func SelectTargetStores(stores []*core.StoreInfo, filters []Filter, opt *config.PersistOptions, collector *plan.Collector) []*core.StoreInfo { return filterStoresBy(stores, func(s *core.StoreInfo) bool { @@ -829,41 +852,3 @@ func createRegionForRuleFit(startKey, endKey []byte, }, copyLeader, opts...) return cloneRegion } - -// RegionScoreFilter filter target store that it's score must higher than the given score -type RegionScoreFilter struct { - scope string - score float64 -} - -// NewRegionScoreFilter creates a Filter that filters all high score stores. -func NewRegionScoreFilter(scope string, source *core.StoreInfo, opt *config.PersistOptions) Filter { - return &RegionScoreFilter{ - scope: scope, - score: source.RegionScore(opt.GetRegionScoreFormulaVersion(), opt.GetHighSpaceRatio(), opt.GetLowSpaceRatio(), 0), - } -} - -// Scope scopes only for balance region -func (f *RegionScoreFilter) Scope() string { - return f.scope -} - -// Type types region score filter -func (f *RegionScoreFilter) Type() string { - return "region-score-filter" -} - -// Source ignore source -func (f *RegionScoreFilter) Source(opt *config.PersistOptions, _ *core.StoreInfo) *plan.Status { - return statusOK -} - -// Target return true if target's score less than source's score -func (f *RegionScoreFilter) Target(opt *config.PersistOptions, store *core.StoreInfo) *plan.Status { - score := store.RegionScore(opt.GetRegionScoreFormulaVersion(), opt.GetHighSpaceRatio(), opt.GetLowSpaceRatio(), 0) - if score < f.score { - return statusOK - } - return statusStoreScoreDisallowed -} diff --git a/server/schedule/operator/builder.go b/server/schedule/operator/builder.go index 8fd2a8a2386d..8900a63aa2f8 100644 --- a/server/schedule/operator/builder.go +++ b/server/schedule/operator/builder.go @@ -218,9 +218,10 @@ func (b *Builder) PromoteLearner(storeID uint64) *Builder { b.err = errors.Errorf("cannot promote peer %d: unhealthy", storeID) } else { b.targetPeers.Set(&metapb.Peer{ - Id: peer.GetId(), - StoreId: peer.GetStoreId(), - Role: metapb.PeerRole_Voter, + Id: peer.GetId(), + StoreId: peer.GetStoreId(), + Role: metapb.PeerRole_Voter, + IsWitness: peer.GetIsWitness(), }) } return b @@ -237,9 +238,30 @@ func (b *Builder) DemoteVoter(storeID uint64) *Builder { b.err = errors.Errorf("cannot demote voter %d: is already learner", storeID) } else { b.targetPeers.Set(&metapb.Peer{ - Id: peer.GetId(), - StoreId: peer.GetStoreId(), - Role: metapb.PeerRole_Learner, + Id: peer.GetId(), + StoreId: peer.GetStoreId(), + Role: metapb.PeerRole_Learner, + IsWitness: peer.GetIsWitness(), + }) + } + return b +} + +// BecomeNonWitness records a remove witness attr operation in Builder. +func (b *Builder) BecomeNonWitness(storeID uint64) *Builder { + if b.err != nil { + return b + } + if peer, ok := b.targetPeers[storeID]; !ok { + b.err = errors.Errorf("cannot set non-witness attr to peer %d: not found", storeID) + } else if !core.IsWitness(peer) { + b.err = errors.Errorf("cannot set non-witness attr to peer %d: is already non-witness", storeID) + } else { + b.targetPeers.Set(&metapb.Peer{ + Id: peer.GetId(), + StoreId: peer.GetStoreId(), + Role: metapb.PeerRole_Learner, + IsWitness: false, }) } return b @@ -404,9 +426,10 @@ func (b *Builder) prepareBuild() (string, error) { // modify it to the peer id of the origin. if o.GetId() != n.GetId() { n = &metapb.Peer{ - Id: o.GetId(), - StoreId: o.GetStoreId(), - Role: n.GetRole(), + Id: o.GetId(), + StoreId: o.GetStoreId(), + Role: n.GetRole(), + IsWitness: n.GetIsWitness(), } } @@ -436,9 +459,10 @@ func (b *Builder) prepareBuild() (string, error) { return "", err } n = &metapb.Peer{ - Id: id, - StoreId: n.GetStoreId(), - Role: n.GetRole(), + Id: id, + StoreId: n.GetStoreId(), + Role: n.GetRole(), + IsWitness: n.GetIsWitness(), } } // It is a pair with `b.toRemove.Set(o)` when `o != nil`. @@ -505,9 +529,10 @@ func (b *Builder) buildStepsWithJointConsensus(kind OpKind) (OpKind, error) { peer := b.toAdd[add] if !core.IsLearner(peer) { b.execAddPeer(&metapb.Peer{ - Id: peer.GetId(), - StoreId: peer.GetStoreId(), - Role: metapb.PeerRole_Learner, + Id: peer.GetId(), + StoreId: peer.GetStoreId(), + Role: metapb.PeerRole_Learner, + IsWitness: peer.GetIsWitness(), }) b.toPromote.Set(peer) } else { @@ -526,9 +551,10 @@ func (b *Builder) buildStepsWithJointConsensus(kind OpKind) (OpKind, error) { peer := b.toRemove[remove] if !core.IsLearner(peer) { b.toDemote.Set(&metapb.Peer{ - Id: peer.GetId(), - StoreId: peer.GetStoreId(), - Role: metapb.PeerRole_Learner, + Id: peer.GetId(), + StoreId: peer.GetStoreId(), + Role: metapb.PeerRole_Learner, + IsWitness: peer.GetIsWitness(), }) } } @@ -675,19 +701,19 @@ func (b *Builder) execTransferLeader(targetStoreID uint64, targetStoreIDs []uint } func (b *Builder) execPromoteLearner(peer *metapb.Peer) { - b.steps = append(b.steps, PromoteLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId()}) + b.steps = append(b.steps, PromoteLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsWitness: peer.GetIsWitness()}) b.currentPeers.Set(peer) delete(b.toPromote, peer.GetStoreId()) } func (b *Builder) execAddPeer(peer *metapb.Peer) { if b.lightWeight { - b.steps = append(b.steps, AddLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsLightWeight: b.lightWeight}) + b.steps = append(b.steps, AddLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsLightWeight: b.lightWeight, IsWitness: peer.GetIsWitness()}) } else { - b.steps = append(b.steps, AddLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId()}) + b.steps = append(b.steps, AddLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsWitness: peer.GetIsWitness()}) } if !core.IsLearner(peer) { - b.steps = append(b.steps, PromoteLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId()}) + b.steps = append(b.steps, PromoteLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsWitness: peer.GetIsWitness()}) } b.currentPeers.Set(peer) b.peerAddStep[peer.GetStoreId()] = len(b.steps) @@ -733,7 +759,7 @@ func (b *Builder) execChangePeerV2(needEnter bool, needTransferLeader bool) { for _, d := range b.toDemote.IDs() { peer := b.toDemote[d] - step.DemoteVoters = append(step.DemoteVoters, DemoteVoter{ToStore: peer.GetStoreId(), PeerID: peer.GetId()}) + step.DemoteVoters = append(step.DemoteVoters, DemoteVoter{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsWitness: peer.GetIsWitness()}) b.currentPeers.Set(peer) } b.toDemote = newPeersMap() diff --git a/server/schedule/operator/create_operator.go b/server/schedule/operator/create_operator.go index 27a1b2647756..ef88cb397a15 100644 --- a/server/schedule/operator/create_operator.go +++ b/server/schedule/operator/create_operator.go @@ -76,12 +76,20 @@ func CreateForceTransferLeaderOperator(desc string, ci ClusterInformer, region * // CreateMoveRegionOperator creates an operator that moves a region to specified stores. func CreateMoveRegionOperator(desc string, ci ClusterInformer, region *core.RegionInfo, kind OpKind, roles map[uint64]placement.PeerRoleType) (*Operator, error) { // construct the peers from roles + oldPeers := region.GetPeers() peers := make(map[uint64]*metapb.Peer) + i := 0 for storeID, role := range roles { + isWitness := false + if i < len(oldPeers) { + isWitness = oldPeers[i].GetIsWitness() + } peers[storeID] = &metapb.Peer{ - StoreId: storeID, - Role: role.MetaPeerRole(), + StoreId: storeID, + Role: role.MetaPeerRole(), + IsWitness: isWitness, } + i += 1 } builder := NewBuilder(desc, ci, region).SetPeers(peers).SetExpectedRoles(roles) return builder.Build(kind) @@ -282,3 +290,15 @@ func CreateLeaveJointStateOperator(desc string, ci ClusterInformer, origin *core b.execChangePeerV2(false, true) return NewOperator(b.desc, brief, b.regionID, b.regionEpoch, kind, origin.GetApproximateSize(), b.steps...), nil } + +// CreateWitnessPeerOperator creates an operator that set a follower or learner peer with witness +func CreateWitnessPeerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { + brief := fmt.Sprintf("create witness: region %v peer %v on store %v", region.GetID(), peer.Id, peer.StoreId) + return NewOperator(desc, brief, region.GetID(), region.GetRegionEpoch(), OpRegion, region.GetApproximateSize(), BecomeWitness{StoreID: peer.StoreId, PeerID: peer.Id}), nil +} + +// CreateNonWitnessPeerOperator creates an operator that set a peer with non-witness +func CreateNonWitnessPeerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { + brief := fmt.Sprintf("promote to non-witness: region %v peer %v on store %v", region.GetID(), peer.Id, peer.StoreId) + return NewOperator(desc, brief, region.GetID(), region.GetRegionEpoch(), OpRegion, region.GetApproximateSize(), BecomeNonWitness{StoreID: peer.StoreId, PeerID: peer.Id}), nil +} diff --git a/server/schedule/operator/operator.go b/server/schedule/operator/operator.go index 9d1d1e50186f..62f20104538a 100644 --- a/server/schedule/operator/operator.go +++ b/server/schedule/operator/operator.go @@ -60,9 +60,9 @@ type Operator struct { // NewOperator creates a new operator. func NewOperator(desc, brief string, regionID uint64, regionEpoch *metapb.RegionEpoch, kind OpKind, approximateSize int64, steps ...OpStep) *Operator { - level := core.NormalPriority + level := core.Medium if kind&OpAdmin != 0 { - level = core.HighPriority + level = core.Urgent } return &Operator{ desc: desc, diff --git a/server/schedule/operator/operator_test.go b/server/schedule/operator/operator_test.go index aeb9286ebdc8..851f2ef5d142 100644 --- a/server/schedule/operator/operator_test.go +++ b/server/schedule/operator/operator_test.go @@ -114,7 +114,7 @@ func (suite *operatorTestSuite) TestOperator() { RemovePeer{FromStore: 3}, } op := suite.newTestOperator(1, OpAdmin|OpLeader|OpRegion, steps...) - suite.Equal(core.HighPriority, op.GetPriorityLevel()) + suite.Equal(core.Urgent, op.GetPriorityLevel()) suite.checkSteps(op, steps) op.Start() suite.Nil(op.Check(region)) @@ -130,6 +130,7 @@ func (suite *operatorTestSuite) TestOperator() { RemovePeer{FromStore: 2}, } op = suite.newTestOperator(1, OpLeader|OpRegion, steps...) + suite.Equal(core.Medium, op.GetPriorityLevel()) suite.checkSteps(op, steps) op.Start() suite.Equal(RemovePeer{FromStore: 2}, op.Check(region)) diff --git a/server/schedule/operator/step.go b/server/schedule/operator/step.go index 8acbf15c5b79..8374f8362ff8 100644 --- a/server/schedule/operator/step.go +++ b/server/schedule/operator/step.go @@ -134,6 +134,7 @@ func (tl TransferLeader) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) * type AddPeer struct { ToStore, PeerID uint64 IsLightWeight bool + IsWitness bool } // ConfVerChanged returns the delta value for version increased by this step. @@ -153,6 +154,9 @@ func (ap AddPeer) IsFinish(region *core.RegionInfo) bool { log.Warn("obtain unexpected peer", zap.String("expect", ap.String()), zap.Uint64("obtain-voter", peer.GetId())) return false } + if peer.GetIsWitness() != ap.IsWitness { + return false + } return region.GetPendingVoter(peer.GetId()) == nil } return false @@ -195,13 +199,132 @@ func (ap AddPeer) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *pdpb.Re // The newly added peer is pending. return nil } - return createResponse(addNode(ap.PeerID, ap.ToStore), useConfChangeV2) + return createResponse(addNode(ap.PeerID, ap.ToStore, ap.IsWitness), useConfChangeV2) +} + +// BecomeWitness is an OpStep that makes a peer become a witness. +type BecomeWitness struct { + StoreID, PeerID uint64 +} + +// ConfVerChanged returns the delta value for version increased by this step. +func (bw BecomeWitness) ConfVerChanged(region *core.RegionInfo) uint64 { + peer := region.GetStorePeer(bw.StoreID) + return typeutil.BoolToUint64(peer.GetId() == bw.PeerID) +} + +func (bw BecomeWitness) String() string { + return fmt.Sprintf("change peer %v on store %v to witness", bw.PeerID, bw.StoreID) +} + +// IsFinish checks if current step is finished. +func (bw BecomeWitness) IsFinish(region *core.RegionInfo) bool { + if peer := region.GetStorePeer(bw.StoreID); peer != nil { + if peer.GetId() != bw.PeerID { + log.Warn("obtain unexpected peer", zap.String("expect", bw.String()), zap.Uint64("obtain-learner", peer.GetId())) + return false + } + return peer.IsWitness + } + return false +} + +// CheckInProgress checks if the step is in the progress of advancing. +func (bw BecomeWitness) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { + if err := validateStore(ci, bw.StoreID); err != nil { + return err + } + peer := region.GetStorePeer(bw.StoreID) + if peer == nil || peer.GetId() != bw.PeerID { + return errors.New("peer does not exist") + } + return nil +} + +// Influence calculates the store difference that current step makes. +func (bw BecomeWitness) Influence(opInfluence OpInfluence, region *core.RegionInfo) { + to := opInfluence.GetStoreInfluence(bw.StoreID) + + regionSize := region.GetApproximateSize() + to.RegionSize -= regionSize + to.AdjustStepCost(storelimit.RemovePeer, regionSize) +} + +// Timeout returns true if the step is timeout. +func (bw BecomeWitness) Timeout(start time.Time, regionSize int64) bool { + return time.Since(start) > fastStepWaitDuration(regionSize) +} + +// GetCmd returns the schedule command for heartbeat response. +func (bw BecomeWitness) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *pdpb.RegionHeartbeatResponse { + if core.IsLearner(region.GetStorePeer(bw.StoreID)) { + return createResponse(addLearnerNode(bw.PeerID, bw.StoreID, true), useConfChangeV2) + } + return createResponse(addNode(bw.PeerID, bw.StoreID, true), useConfChangeV2) +} + +// BecomeNonWitness is an OpStep that makes a peer become a non-witness. +type BecomeNonWitness struct { + StoreID, PeerID uint64 +} + +// ConfVerChanged returns the delta value for version increased by this step. +func (bn BecomeNonWitness) ConfVerChanged(region *core.RegionInfo) uint64 { + peer := region.GetStorePeer(bn.StoreID) + return typeutil.BoolToUint64(peer.GetId() == bn.PeerID) +} + +func (bn BecomeNonWitness) String() string { + return fmt.Sprintf("change peer %v on store %v to non-witness", bn.PeerID, bn.StoreID) +} + +// IsFinish checks if current step is finished. +func (bn BecomeNonWitness) IsFinish(region *core.RegionInfo) bool { + if peer := region.GetStorePeer(bn.StoreID); peer != nil { + if peer.GetId() != bn.PeerID { + log.Warn("obtain unexpected peer", zap.String("expect", bn.String()), zap.Uint64("obtain-non-witness", peer.GetId())) + return false + } + return region.GetPendingPeer(peer.GetId()) == nil && !peer.IsWitness + } + return false +} + +// CheckInProgress checks if the step is in the progress of advancing. +func (bn BecomeNonWitness) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { + if err := validateStore(ci, bn.StoreID); err != nil { + return err + } + peer := region.GetStorePeer(bn.StoreID) + if peer == nil || peer.GetId() != bn.PeerID { + return errors.New("peer does not exist") + } + return nil +} + +// Influence calculates the store difference that current step makes. +func (bn BecomeNonWitness) Influence(opInfluence OpInfluence, region *core.RegionInfo) { + to := opInfluence.GetStoreInfluence(bn.StoreID) + + regionSize := region.GetApproximateSize() + to.AdjustStepCost(storelimit.AddPeer, regionSize) +} + +// Timeout returns true if the step is timeout +func (bn BecomeNonWitness) Timeout(start time.Time, regionSize int64) bool { + return time.Since(start) > slowStepWaitDuration(regionSize) +} + +// GetCmd returns the schedule command for heartbeat response. +func (bn BecomeNonWitness) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *pdpb.RegionHeartbeatResponse { + return createResponse(addLearnerNode(bn.PeerID, bn.StoreID, false), useConfChangeV2) } // AddLearner is an OpStep that adds a region learner peer. type AddLearner struct { ToStore, PeerID uint64 IsLightWeight bool + IsWitness bool } // ConfVerChanged returns the delta value for version increased by this step. @@ -221,6 +344,9 @@ func (al AddLearner) IsFinish(region *core.RegionInfo) bool { log.Warn("obtain unexpected peer", zap.String("expect", al.String()), zap.Uint64("obtain-learner", peer.GetId())) return false } + if peer.GetIsWitness() != al.IsWitness { + return false + } return region.GetPendingLearner(peer.GetId()) == nil } return false @@ -268,12 +394,13 @@ func (al AddLearner) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *pdpb // The newly added peer is pending. return nil } - return createResponse(addLearnerNode(al.PeerID, al.ToStore), useConfChangeV2) + return createResponse(addLearnerNode(al.PeerID, al.ToStore, al.IsWitness), useConfChangeV2) } // PromoteLearner is an OpStep that promotes a region learner peer to normal voter. type PromoteLearner struct { ToStore, PeerID uint64 + IsWitness bool } // ConfVerChanged returns the delta value for version increased by this step. @@ -316,7 +443,7 @@ func (pl PromoteLearner) Timeout(start time.Time, regionSize int64) bool { // GetCmd returns the schedule command for heartbeat response. func (pl PromoteLearner) GetCmd(_ *core.RegionInfo, useConfChangeV2 bool) *pdpb.RegionHeartbeatResponse { - return createResponse(addNode(pl.PeerID, pl.ToStore), useConfChangeV2) + return createResponse(addNode(pl.PeerID, pl.ToStore, pl.IsWitness), useConfChangeV2) } // RemovePeer is an OpStep that removes a region peer. @@ -504,10 +631,15 @@ func (sr SplitRegion) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *pdp // Note: It is not an OpStep, only a sub step in ChangePeerV2Enter and ChangePeerV2Leave. type DemoteVoter struct { ToStore, PeerID uint64 + IsWitness bool } func (dv DemoteVoter) String() string { - return fmt.Sprintf("demote voter peer %v on store %v to learner", dv.PeerID, dv.ToStore) + info := "non-witness" + if dv.IsWitness { + info = "witness" + } + return fmt.Sprintf("demote voter peer %v on store %v to %v learner", dv.PeerID, dv.ToStore, info) } // ConfVerChanged returns the delta value for version increased by this step. @@ -521,8 +653,12 @@ func (dv DemoteVoter) IsFinish(region *core.RegionInfo) bool { if peer := region.GetStoreLearner(dv.ToStore); peer != nil { if peer.GetId() != dv.PeerID { log.Warn("obtain unexpected peer", zap.String("expect", dv.String()), zap.Uint64("obtain-learner", peer.GetId())) + return false + } + if peer.IsWitness != dv.IsWitness { + return false } - return peer.GetId() == dv.PeerID + return region.GetPendingLearner(peer.GetId()) == nil } return false } @@ -534,7 +670,7 @@ func (dv DemoteVoter) Timeout(start time.Time, regionSize int64) bool { // GetCmd returns the schedule command for heartbeat response. func (dv DemoteVoter) GetCmd(_ *core.RegionInfo, useConfChangeV2 bool) *pdpb.RegionHeartbeatResponse { - return createResponse(addLearnerNode(dv.PeerID, dv.ToStore), useConfChangeV2) + return createResponse(addLearnerNode(dv.PeerID, dv.ToStore, dv.IsWitness), useConfChangeV2) } // ChangePeerV2Enter is an OpStep that uses joint consensus to request all PromoteLearner and DemoteVoter. @@ -841,24 +977,26 @@ func fastStepWaitDuration(regionSize int64) time.Duration { return wait } -func addNode(id, storeID uint64) *pdpb.ChangePeer { +func addNode(id, storeID uint64, isWitness bool) *pdpb.ChangePeer { return &pdpb.ChangePeer{ ChangeType: eraftpb.ConfChangeType_AddNode, Peer: &metapb.Peer{ - Id: id, - StoreId: storeID, - Role: metapb.PeerRole_Voter, + Id: id, + StoreId: storeID, + Role: metapb.PeerRole_Voter, + IsWitness: isWitness, }, } } -func addLearnerNode(id, storeID uint64) *pdpb.ChangePeer { +func addLearnerNode(id, storeID uint64, isWitness bool) *pdpb.ChangePeer { return &pdpb.ChangePeer{ ChangeType: eraftpb.ConfChangeType_AddLearnerNode, Peer: &metapb.Peer{ - Id: id, - StoreId: storeID, - Role: metapb.PeerRole_Learner, + Id: id, + StoreId: storeID, + Role: metapb.PeerRole_Learner, + IsWitness: isWitness, }, } } diff --git a/server/schedule/operator_controller.go b/server/schedule/operator_controller.go index 65bb41b53897..4fd806ee28ca 100644 --- a/server/schedule/operator_controller.go +++ b/server/schedule/operator_controller.go @@ -831,6 +831,10 @@ func (oc *OperatorController) ExceedStoreLimit(ops ...*operator.Operator) bool { // exceedStoreLimitLocked returns true if the store exceeds the cost limit after adding the operator. Otherwise, returns false. func (oc *OperatorController) exceedStoreLimitLocked(ops ...*operator.Operator) bool { + // The operator with Urgent priority, like admin operators, should ignore the store limit check. + if len(ops) != 0 && ops[0].GetPriorityLevel() == core.Urgent { + return false + } opInfluence := NewTotalOpInfluence(ops, oc.cluster) for storeID := range opInfluence.StoresInfluence { for _, v := range storelimit.TypeNameValue { diff --git a/server/schedule/placement/fit.go b/server/schedule/placement/fit.go index 54f6c53bd90d..33e0af2dbc95 100644 --- a/server/schedule/placement/fit.go +++ b/server/schedule/placement/fit.go @@ -385,7 +385,7 @@ func newRuleFit(rule *Rule, peers []*fitPeer) *RuleFit { rf := &RuleFit{Rule: rule, IsolationScore: isolationScore(peers, rule.LocationLabels)} for _, p := range peers { rf.Peers = append(rf.Peers, p.Peer) - if !p.matchRoleStrict(rule.Role) { + if !p.matchRoleStrict(rule.Role) || p.IsWitness != rule.IsWitness { rf.PeersWithDifferentRole = append(rf.PeersWithDifferentRole, p.Peer) } } diff --git a/server/schedule/placement/rule.go b/server/schedule/placement/rule.go index 31493b908bd1..5b8b488c6af1 100644 --- a/server/schedule/placement/rule.go +++ b/server/schedule/placement/rule.go @@ -63,6 +63,7 @@ type Rule struct { EndKey []byte `json:"-"` // range end key EndKeyHex string `json:"end_key"` // hex format end key, for marshal/unmarshal Role PeerRoleType `json:"role"` // expected role of the peers + IsWitness bool `json:"is_witness"` // when it is true, it means the role is also a witness Count int `json:"count"` // expected count of the peers LabelConstraints []LabelConstraint `json:"label_constraints,omitempty"` // used to select stores to place peers LocationLabels []string `json:"location_labels,omitempty"` // used to make peers isolated physically diff --git a/server/schedule/plan/plan.go b/server/schedule/plan/plan.go index 2cc0607187f8..fcd5102012c8 100644 --- a/server/schedule/plan/plan.go +++ b/server/schedule/plan/plan.go @@ -23,6 +23,10 @@ type Plan interface { Clone(ops ...Option) Plan // generate plan for clone option SetResource(interface{}) + // SetResourceWithStep is used to set resource for specific step. + // The meaning of step is different for different plans. + // Such as balancePlan, pickSource = 0, pickRegion = 1, pickTarget = 2 + SetResourceWithStep(resource interface{}, step int) SetStatus(*Status) } @@ -83,3 +87,10 @@ func SetResource(resource interface{}) Option { plan.SetResource(resource) } } + +// SetResourceWithStep is used to generate Resource for plan +func SetResourceWithStep(resource interface{}, step int) Option { + return func(plan Plan) { + plan.SetResourceWithStep(resource, step) + } +} diff --git a/server/schedule/region_scatterer.go b/server/schedule/region_scatterer.go index 0dec30ae2325..d3a610a32743 100644 --- a/server/schedule/region_scatterer.go +++ b/server/schedule/region_scatterer.go @@ -372,7 +372,7 @@ func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string) * if op != nil { scatterCounter.WithLabelValues("success", "").Inc() r.Put(targetPeers, targetLeader, group) - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) } return op } diff --git a/server/schedule/waiting_operator.go b/server/schedule/waiting_operator.go index e10848bd4625..513f8edea1ae 100644 --- a/server/schedule/waiting_operator.go +++ b/server/schedule/waiting_operator.go @@ -22,7 +22,7 @@ import ( ) // PriorityWeight is used to represent the weight of different priorities of operators. -var PriorityWeight = []float64{1.0, 4.0, 9.0} +var PriorityWeight = []float64{1.0, 4.0, 9.0, 16.0} // WaitingOperator is an interface of waiting operators. type WaitingOperator interface { diff --git a/server/schedule/waiting_operator_test.go b/server/schedule/waiting_operator_test.go index bc7c37936db9..8c639c1568ca 100644 --- a/server/schedule/waiting_operator_test.go +++ b/server/schedule/waiting_operator_test.go @@ -27,7 +27,7 @@ func TestRandBuckets(t *testing.T) { re := require.New(t) rb := NewRandBuckets() addOperators(rb) - for i := 0; i < 3; i++ { + for i := 0; i < len(PriorityWeight); i++ { op := rb.GetOperator() re.NotNil(op) } @@ -38,16 +38,22 @@ func addOperators(wop WaitingOperator) { op := operator.NewTestOperator(uint64(1), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ operator.RemovePeer{FromStore: uint64(1)}, }...) + op.SetPriorityLevel(core.Medium) wop.PutOperator(op) op = operator.NewTestOperator(uint64(2), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ operator.RemovePeer{FromStore: uint64(2)}, }...) - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) wop.PutOperator(op) op = operator.NewTestOperator(uint64(3), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ operator.RemovePeer{FromStore: uint64(3)}, }...) - op.SetPriorityLevel(core.LowPriority) + op.SetPriorityLevel(core.Low) + wop.PutOperator(op) + op = operator.NewTestOperator(uint64(4), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ + operator.RemovePeer{FromStore: uint64(4)}, + }...) + op.SetPriorityLevel(core.Urgent) wop.PutOperator(op) } @@ -55,7 +61,7 @@ func TestListOperator(t *testing.T) { re := require.New(t) rb := NewRandBuckets() addOperators(rb) - re.Len(rb.ListOperator(), 3) + re.Len(rb.ListOperator(), len(PriorityWeight)) } func TestRandomBucketsWithMergeRegion(t *testing.T) { @@ -101,7 +107,7 @@ func TestRandomBucketsWithMergeRegion(t *testing.T) { operator.RemovePeer{FromStore: uint64(3)}, }...) op.SetDesc("testOperatorHigh") - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) rb.PutOperator(op) for i := 0; i < 2; i++ { diff --git a/server/schedulers/balance_leader.go b/server/schedulers/balance_leader.go index 7490b8b9ba35..4cb542073ee2 100644 --- a/server/schedulers/balance_leader.go +++ b/server/schedulers/balance_leader.go @@ -540,6 +540,7 @@ func (l *balanceLeaderScheduler) transferLeaderIn(solver *solver, collector *pla func (l *balanceLeaderScheduler) createOperator(solver *solver, collector *plan.Collector) *operator.Operator { solver.step++ defer func() { solver.step-- }() + solver.sourceScore, solver.targetScore = solver.sourceStoreScore(l.GetName()), solver.targetStoreScore(l.GetName()) if !solver.shouldBalance(l.GetName()) { schedulerCounter.WithLabelValues(l.GetName(), "skip").Inc() if collector != nil { diff --git a/server/schedulers/balance_plan.go b/server/schedulers/balance_plan.go index 306c0e52a95d..421b24ab9acf 100644 --- a/server/schedulers/balance_plan.go +++ b/server/schedulers/balance_plan.go @@ -63,6 +63,11 @@ func (p *balanceSchedulerPlan) SetResource(resource interface{}) { } } +func (p *balanceSchedulerPlan) SetResourceWithStep(resource interface{}, step int) { + p.step = step + p.SetResource(resource) +} + func (p *balanceSchedulerPlan) GetResource(step int) uint64 { if p.step < step { return 0 diff --git a/server/schedulers/balance_plan_test.go b/server/schedulers/balance_plan_test.go index 6e6debffdef4..266dc22b60c8 100644 --- a/server/schedulers/balance_plan_test.go +++ b/server/schedulers/balance_plan_test.go @@ -248,3 +248,27 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult5() { 5: plan.NewStatus(plan.StatusCreateOperatorFailed), })) } + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult6() { + basePlan := NewBalanceSchedulerPlan() + collector := plan.NewCollector(basePlan) + collector.Collect(plan.SetResourceWithStep(suite.stores[0], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) + collector.Collect(plan.SetResourceWithStep(suite.stores[1], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) + collector.Collect(plan.SetResourceWithStep(suite.stores[2], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) + collector.Collect(plan.SetResourceWithStep(suite.stores[3], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) + collector.Collect(plan.SetResourceWithStep(suite.stores[4], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) + basePlan.source = suite.stores[0] + basePlan.step++ + collector.Collect(plan.SetResource(suite.regions[0]), plan.SetStatus(plan.NewStatus(plan.StatusRegionNoLeader))) + statuses, isNormal, err := BalancePlanSummary(collector.GetPlans()) + suite.NoError(err) + suite.False(isNormal) + suite.True(suite.check(statuses, + map[uint64]*plan.Status{ + 1: plan.NewStatus(plan.StatusStoreDown), + 2: plan.NewStatus(plan.StatusStoreDown), + 3: plan.NewStatus(plan.StatusStoreDown), + 4: plan.NewStatus(plan.StatusStoreDown), + 5: plan.NewStatus(plan.StatusStoreDown), + })) +} diff --git a/server/schedulers/balance_region.go b/server/schedulers/balance_region.go index ecaccc9fb09e..fc211ffe3f0b 100644 --- a/server/schedulers/balance_region.go +++ b/server/schedulers/balance_region.go @@ -146,17 +146,18 @@ func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) schedulerCounter.WithLabelValues(s.GetName(), "schedule").Inc() stores := cluster.GetStores() opts := cluster.GetOpts() - stores = filter.SelectSourceStores(stores, s.filters, opts, collector) + faultTargets := filter.SelectUnavailableTargetStores(stores, s.filters, opts, collector) + sourceStores := filter.SelectSourceStores(stores, s.filters, opts, collector) opInfluence := s.opController.GetOpInfluence(cluster) s.OpController.GetFastOpInfluence(cluster, opInfluence) kind := core.NewScheduleKind(core.RegionKind, core.BySize) solver := newSolver(basePlan, kind, cluster, opInfluence) - sort.Slice(stores, func(i, j int) bool { - iOp := solver.GetOpInfluence(stores[i].GetID()) - jOp := solver.GetOpInfluence(stores[j].GetID()) - return stores[i].RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), iOp) > - stores[j].RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), jOp) + sort.Slice(sourceStores, func(i, j int) bool { + iOp := solver.GetOpInfluence(sourceStores[i].GetID()) + jOp := solver.GetOpInfluence(sourceStores[j].GetID()) + return sourceStores[i].RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), iOp) > + sourceStores[j].RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), jOp) }) pendingFilter := filter.NewRegionPendingFilter() @@ -171,8 +172,15 @@ func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) } solver.step++ - for _, solver.source = range stores { + var sourceIndex int + + // sourcesStore is sorted by region score desc, so we pick the first store as source store. + for sourceIndex, solver.source = range sourceStores { retryLimit := s.retryQuota.GetLimit(solver.source) + solver.sourceScore = solver.sourceStoreScore(s.GetName()) + if sourceIndex == len(sourceStores)-1 { + break + } for i := 0; i < retryLimit; i++ { schedulerCounter.WithLabelValues(s.GetName(), "total").Inc() // Priority pick the region that has a pending peer. @@ -218,7 +226,7 @@ func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) continue } solver.step++ - if op := s.transferPeer(solver, collector); op != nil { + if op := s.transferPeer(solver, collector, sourceStores[sourceIndex+1:], faultTargets); op != nil { s.retryQuota.ResetLimit(solver.source) op.Counters = append(op.Counters, schedulerCounter.WithLabelValues(s.GetName(), "new-operator")) return []*operator.Operator{op}, collector.GetPlans() @@ -232,25 +240,27 @@ func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) } // transferPeer selects the best store to create a new peer to replace the old peer. -func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Collector) *operator.Operator { +func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Collector, dstStores []*core.StoreInfo, faultStores []*core.StoreInfo) *operator.Operator { + excludeTargets := solver.region.GetStoreIDs() + for _, store := range faultStores { + excludeTargets[store.GetID()] = struct{}{} + } // the order of the filters should be sorted by the cost of the cpu overhead. // the more expensive the filter is, the later it should be placed. filters := []filter.Filter{ - filter.NewExcludedFilter(s.GetName(), nil, solver.region.GetStoreIDs()), - filter.NewSpecialUseFilter(s.GetName()), - &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true}, - filter.NewRegionScoreFilter(s.GetName(), solver.source, solver.GetOpts()), + filter.NewExcludedFilter(s.GetName(), nil, excludeTargets), filter.NewPlacementSafeguard(s.GetName(), solver.GetOpts(), solver.GetBasicCluster(), solver.GetRuleManager(), solver.region, solver.source), } - candidates := filter.NewCandidates(solver.GetStores()). - FilterTarget(solver.GetOpts(), collector, filters...). - Sort(filter.RegionScoreComparer(solver.GetOpts())) - + candidates := filter.NewCandidates(dstStores).FilterTarget(solver.GetOpts(), collector, filters...) if len(candidates.Stores) != 0 { solver.step++ } - for _, solver.target = range candidates.Stores { + + // candidates are sorted by region score desc, so we pick the last store as target store. + for i := range candidates.Stores { + solver.target = candidates.Stores[len(candidates.Stores)-i-1] + solver.targetScore = solver.targetStoreScore(s.GetName()) regionID := solver.region.GetID() sourceID := solver.source.GetID() targetID := solver.target.GetID() diff --git a/server/schedulers/balance_test.go b/server/schedulers/balance_test.go index 0b7a72f93e2e..84499fbf3e17 100644 --- a/server/schedulers/balance_test.go +++ b/server/schedulers/balance_test.go @@ -71,12 +71,14 @@ func TestInfluenceAmp(t *testing.T) { basePlan := NewBalanceSchedulerPlan() solver := newSolver(basePlan, kind, tc, influence) solver.source, solver.target, solver.region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) + solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.True(solver.shouldBalance("")) // It will not schedule if the diff region count is greater than the sum // of TolerantSizeRatio and influenceAmp*2. tc.AddRegionStore(1, int(100+influenceAmp+2)) solver.source = tc.GetStore(1) + solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.False(solver.shouldBalance("")) re.Less(solver.sourceScore-solver.targetScore, float64(1)) } @@ -157,6 +159,7 @@ func TestShouldBalance(t *testing.T) { basePlan := NewBalanceSchedulerPlan() solver := newSolver(basePlan, kind, tc, oc.GetOpInfluence(tc)) solver.source, solver.target, solver.region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) + solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.Equal(testCase.expectedResult, solver.shouldBalance("")) } @@ -170,6 +173,7 @@ func TestShouldBalance(t *testing.T) { basePlan := NewBalanceSchedulerPlan() solver := newSolver(basePlan, kind, tc, oc.GetOpInfluence(tc)) solver.source, solver.target, solver.region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) + solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.Equal(testCase.expectedResult, solver.shouldBalance("")) } } @@ -184,7 +188,7 @@ func TestTolerantRatio(t *testing.T) { tc := mockcluster.NewCluster(ctx, opt) // create a region to control average region size. re.NotNil(tc.AddLeaderRegion(1, 1, 2)) - regionSize := int64(96 * units.MiB) + regionSize := int64(96) region := tc.GetRegion(1).Clone(core.SetApproximateSize(regionSize)) tbl := []struct { @@ -222,7 +226,10 @@ func TestTolerantRatio(t *testing.T) { basePlan := NewBalanceSchedulerPlan() solver := newSolver(basePlan, t.kind, tc, operator.OpInfluence{}) solver.region = region - re.Equal(t.expectTolerantResource(t.kind), solver.getTolerantResource()) + + sourceScore := t.expectTolerantResource(t.kind) + targetScore := solver.getTolerantResource() + re.Equal(sourceScore, targetScore) } } @@ -776,7 +783,7 @@ func TestBalanceRegionSchedule1(t *testing.T) { ops, _ = sb.Schedule(tc, false) op = ops[0] testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpKind(0), 4, 2) - + tc.SetStoreUp(1) // test region replicate not match opt.SetMaxReplicas(3) ops, plans := sb.Schedule(tc, true) @@ -784,6 +791,7 @@ func TestBalanceRegionSchedule1(t *testing.T) { re.Empty(ops) re.Equal(int(plans[0].GetStatus().StatusCode), plan.StatusRegionNotReplicated) + tc.SetStoreOffline(1) opt.SetMaxReplicas(1) ops, plans = sb.Schedule(tc, true) re.NotEmpty(ops) diff --git a/server/schedulers/evict_leader.go b/server/schedulers/evict_leader.go index 037912237f5b..2e807f18a9aa 100644 --- a/server/schedulers/evict_leader.go +++ b/server/schedulers/evict_leader.go @@ -348,7 +348,7 @@ func scheduleEvictLeaderOnce(name, typ string, cluster schedule.Cluster, conf ev log.Debug("fail to create evict leader operator", errs.ZapError(err)) continue } - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.Urgent) op.Counters = append(op.Counters, schedulerCounter.WithLabelValues(name, "new-operator")) ops = append(ops, op) } diff --git a/server/schedulers/grant_hot_region.go b/server/schedulers/grant_hot_region.go index 61f329480c6b..e2927631f20f 100644 --- a/server/schedulers/grant_hot_region.go +++ b/server/schedulers/grant_hot_region.go @@ -388,5 +388,6 @@ func (s *grantHotRegionScheduler) transfer(cluster schedule.Cluster, regionID ui } else { op, err = operator.CreateMovePeerOperator(GrantHotRegionType+"-move", cluster, srcRegion, operator.OpRegion|operator.OpLeader, srcStore.GetID(), dstStore) } + op.SetPriorityLevel(core.High) return } diff --git a/server/schedulers/grant_leader.go b/server/schedulers/grant_leader.go index 1702bd8f93fd..bac48675323d 100644 --- a/server/schedulers/grant_leader.go +++ b/server/schedulers/grant_leader.go @@ -252,7 +252,7 @@ func (s *grantLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ( continue } op.Counters = append(op.Counters, schedulerCounter.WithLabelValues(s.GetName(), "new-operator")) - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) ops = append(ops, op) } diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index ffcfc483282d..c53dff369ab1 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -182,7 +182,7 @@ func (h *hotScheduler) dispatch(typ statistics.RWType, cluster schedule.Cluster) // each store func (h *hotScheduler) prepareForBalance(typ statistics.RWType, cluster schedule.Cluster) { h.stInfos = statistics.SummaryStoreInfos(cluster.GetStores()) - h.summaryPendingInfluence() + h.summaryPendingInfluence(cluster) storesLoads := cluster.GetStoresLoads() isTraceRegionFlow := cluster.GetOpts().IsTraceRegionFlow() @@ -223,7 +223,7 @@ func (h *hotScheduler) prepareForBalance(typ statistics.RWType, cluster schedule // summaryPendingInfluence calculate the summary of pending Influence for each store // and clean the region from regionInfluence if they have ended operator. // It makes each dim rate or count become `weight` times to the origin value. -func (h *hotScheduler) summaryPendingInfluence() { +func (h *hotScheduler) summaryPendingInfluence(cluster schedule.Cluster) { for id, p := range h.regionPendings { from := h.stInfos[p.from] to := h.stInfos[p.to] @@ -248,6 +248,14 @@ func (h *hotScheduler) summaryPendingInfluence() { to.AddInfluence(&p.origin, weight) } } + for storeID, info := range h.stInfos { + storeLabel := strconv.FormatUint(storeID, 10) + if infl := info.PendingSum; infl != nil { + statistics.ForeachRegionStats(func(rwTy statistics.RWType, dim int, kind statistics.RegionStatKind) { + cluster.SetHotPendingInfluenceMetrics(storeLabel, rwTy.String(), statistics.DimToString(dim), infl.Loads[kind]) + }) + } + } } func (h *hotScheduler) tryAddPendingInfluence(op *operator.Operator, srcStore, dstStore uint64, infl statistics.Influence, maxZombieDur time.Duration) bool { @@ -262,6 +270,9 @@ func (h *hotScheduler) tryAddPendingInfluence(op *operator.Operator, srcStore, d h.regionPendings[regionID] = influence schedulerStatus.WithLabelValues(h.GetName(), "pending_op_infos").Inc() + statistics.ForeachRegionStats(func(rwTy statistics.RWType, dim int, kind statistics.RegionStatKind) { + hotPeerHist.WithLabelValues(h.GetName(), rwTy.String(), statistics.DimToString(dim)).Observe(infl.Loads[kind]) + }) return true } @@ -1383,7 +1394,7 @@ func (bs *balanceSolver) createWriteOperator(region *core.RegionInfo, srcStoreID } func (bs *balanceSolver) decorateOperator(op *operator.Operator, isRevert bool, sourceLabel, targetLabel, typ, dim string) { - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.High) op.FinishedCounters = append(op.FinishedCounters, hotDirectionCounter.WithLabelValues(typ, bs.rwTy.String(), sourceLabel, "out", dim), hotDirectionCounter.WithLabelValues(typ, bs.rwTy.String(), targetLabel, "in", dim), diff --git a/server/schedulers/hot_region_test.go b/server/schedulers/hot_region_test.go index a9995623b3a7..a4f11ac67d53 100644 --- a/server/schedulers/hot_region_test.go +++ b/server/schedulers/hot_region_test.go @@ -121,7 +121,7 @@ func TestGCPendingOpInfos(t *testing.T) { } } - hb.summaryPendingInfluence() // Calling this function will GC. + hb.summaryPendingInfluence(tc) // Calling this function will GC. for i := range opInfluenceCreators { for j, typ := range typs { @@ -1791,7 +1791,7 @@ func TestInfluenceByRWType(t *testing.T) { op := ops[0] re.NotNil(op) - hb.(*hotScheduler).summaryPendingInfluence() + hb.(*hotScheduler).summaryPendingInfluence(tc) stInfos := hb.(*hotScheduler).stInfos re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteKeys], -0.5*units.MiB)) re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteBytes], -0.5*units.MiB)) @@ -1816,7 +1816,7 @@ func TestInfluenceByRWType(t *testing.T) { op = ops[0] re.NotNil(op) - hb.(*hotScheduler).summaryPendingInfluence() + hb.(*hotScheduler).summaryPendingInfluence(tc) stInfos = hb.(*hotScheduler).stInfos // assert read/write influence is the sum of write peer and write leader re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteKeys], -1.2*units.MiB)) diff --git a/server/schedulers/metrics.go b/server/schedulers/metrics.go index fa7e347844e8..21dd8bbe414d 100644 --- a/server/schedulers/metrics.go +++ b/server/schedulers/metrics.go @@ -46,7 +46,7 @@ var tolerantResourceStatus = prometheus.NewGaugeVec( Subsystem: "scheduler", Name: "tolerant_resource", Help: "Store status for schedule", - }, []string{"scheduler", "source", "target"}) + }, []string{"scheduler"}) var balanceLeaderCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -109,9 +109,18 @@ var hotPendingStatus = prometheus.NewGaugeVec( Namespace: "pd", Subsystem: "scheduler", Name: "hot_pending", - Help: "Counter of direction of balance related schedulers.", + Help: "Pending influence status in hot region scheduler.", }, []string{"type", "source", "target"}) +var hotPeerHist = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_peer", + Help: "Bucketed histogram of the scheduling hot peer.", + Buckets: prometheus.ExponentialBuckets(1, 2, 30), + }, []string{"type", "rw", "dim"}) + func init() { prometheus.MustRegister(schedulerCounter) prometheus.MustRegister(schedulerStatus) @@ -125,4 +134,5 @@ func init() { prometheus.MustRegister(opInfluenceStatus) prometheus.MustRegister(tolerantResourceStatus) prometheus.MustRegister(hotPendingStatus) + prometheus.MustRegister(hotPeerHist) } diff --git a/server/schedulers/random_merge.go b/server/schedulers/random_merge.go index 3c942b0b80c6..53c6eb1cf334 100644 --- a/server/schedulers/random_merge.go +++ b/server/schedulers/random_merge.go @@ -137,6 +137,8 @@ func (s *randomMergeScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ( log.Debug("fail to create merge region operator", errs.ZapError(err)) return nil, nil } + ops[0].SetPriorityLevel(core.Low) + ops[1].SetPriorityLevel(core.Low) ops[0].Counters = append(ops[0].Counters, schedulerCounter.WithLabelValues(s.GetName(), "new-operator")) return ops, nil } diff --git a/server/schedulers/shuffle_hot_region.go b/server/schedulers/shuffle_hot_region.go index 072df47ea45b..81fb09510a4e 100644 --- a/server/schedulers/shuffle_hot_region.go +++ b/server/schedulers/shuffle_hot_region.go @@ -209,6 +209,7 @@ func (s *shuffleHotRegionScheduler) randomSchedule(cluster schedule.Cluster, loa log.Debug("fail to create move leader operator", errs.ZapError(err)) return nil } + op.SetPriorityLevel(core.Low) op.Counters = append(op.Counters, schedulerCounter.WithLabelValues(s.GetName(), "new-operator")) return []*operator.Operator{op} } diff --git a/server/schedulers/shuffle_leader.go b/server/schedulers/shuffle_leader.go index 5563c0188278..da9c9777f604 100644 --- a/server/schedulers/shuffle_leader.go +++ b/server/schedulers/shuffle_leader.go @@ -128,7 +128,7 @@ func (s *shuffleLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) log.Debug("fail to create shuffle leader operator", errs.ZapError(err)) return nil, nil } - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.Low) op.Counters = append(op.Counters, schedulerCounter.WithLabelValues(s.GetName(), "new-operator")) return []*operator.Operator{op}, nil } diff --git a/server/schedulers/shuffle_region.go b/server/schedulers/shuffle_region.go index 9d6898009fe2..eeba44ef6afa 100644 --- a/server/schedulers/shuffle_region.go +++ b/server/schedulers/shuffle_region.go @@ -124,7 +124,7 @@ func (s *shuffleRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) return nil, nil } op.Counters = append(op.Counters, schedulerCounter.WithLabelValues(s.GetName(), "new-operator")) - op.SetPriorityLevel(core.HighPriority) + op.SetPriorityLevel(core.Low) return []*operator.Operator{op}, nil } diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index a9eaccd29a01..acd1fbe84f90 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -44,6 +44,7 @@ type solver struct { kind core.ScheduleKind opInfluence operator.OpInfluence tolerantSizeRatio float64 + tolerantSource int64 sourceScore float64 targetScore float64 @@ -79,44 +80,69 @@ func (p *solver) TargetMetricLabel() string { return strconv.FormatUint(p.TargetStoreID(), 10) } -func (p *solver) shouldBalance(scheduleName string) bool { - // The reason we use max(regionSize, averageRegionSize) to check is: - // 1. prevent moving small regions between stores with close scores, leading to unnecessary balance. - // 2. prevent moving huge regions, leading to over balance. +func (p *solver) sourceStoreScore(scheduleName string) float64 { sourceID := p.source.GetID() - targetID := p.target.GetID() tolerantResource := p.getTolerantResource() // to avoid schedule too much, if A's core greater than B and C a little // we want that A should be moved out one region not two - sourceInfluence := p.GetOpInfluence(sourceID) - // A->B, B's influence is positive , so B can become source schedule, it will move region from B to C - if sourceInfluence > 0 { - sourceInfluence = -sourceInfluence + influence := p.GetOpInfluence(sourceID) + if influence > 0 { + influence = -influence + } + + opts := p.GetOpts() + if opts.IsDebugMetricsEnabled() { + opInfluenceStatus.WithLabelValues(scheduleName, strconv.FormatUint(sourceID, 10), "source").Set(float64(influence)) + tolerantResourceStatus.WithLabelValues(scheduleName).Set(float64(tolerantResource)) } + var score float64 + switch p.kind.Resource { + case core.LeaderKind: + sourceDelta := influence - tolerantResource + score = p.source.LeaderScore(p.kind.Policy, sourceDelta) + case core.RegionKind: + sourceDelta := influence*influenceAmp - tolerantResource + score = p.source.RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), sourceDelta) + } + return score +} + +func (p *solver) targetStoreScore(scheduleName string) float64 { + targetID := p.target.GetID() // to avoid schedule too much, if A's score less than B and C in small range, // we want that A can be moved in one region not two - targetInfluence := p.GetOpInfluence(targetID) + tolerantResource := p.getTolerantResource() // to avoid schedule call back // A->B, A's influence is negative, so A will be target, C may move region to A - if targetInfluence < 0 { - targetInfluence = -targetInfluence + influence := p.GetOpInfluence(targetID) + if influence < 0 { + influence = -influence } + opts := p.GetOpts() + if opts.IsDebugMetricsEnabled() { + opInfluenceStatus.WithLabelValues(scheduleName, strconv.FormatUint(targetID, 10), "target").Set(float64(influence)) + } + var score float64 switch p.kind.Resource { case core.LeaderKind: - sourceDelta, targetDelta := sourceInfluence-tolerantResource, targetInfluence+tolerantResource - p.sourceScore = p.source.LeaderScore(p.kind.Policy, sourceDelta) - p.targetScore = p.target.LeaderScore(p.kind.Policy, targetDelta) + targetDelta := influence + tolerantResource + score = p.target.LeaderScore(p.kind.Policy, targetDelta) case core.RegionKind: - sourceDelta, targetDelta := sourceInfluence*influenceAmp-tolerantResource, targetInfluence*influenceAmp+tolerantResource - p.sourceScore = p.source.RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), sourceDelta) - p.targetScore = p.target.RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), targetDelta) - } - if opts.IsDebugMetricsEnabled() { - opInfluenceStatus.WithLabelValues(scheduleName, strconv.FormatUint(sourceID, 10), "source").Set(float64(sourceInfluence)) - opInfluenceStatus.WithLabelValues(scheduleName, strconv.FormatUint(targetID, 10), "target").Set(float64(targetInfluence)) - tolerantResourceStatus.WithLabelValues(scheduleName, strconv.FormatUint(sourceID, 10), strconv.FormatUint(targetID, 10)).Set(float64(tolerantResource)) + targetDelta := influence*influenceAmp + tolerantResource + score = p.target.RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), targetDelta) } + return score +} + +// Both of the source store's score and target store's score should be calculated before calling this function. +// It will not calculate the score again. +func (p *solver) shouldBalance(scheduleName string) bool { + // The reason we use max(regionSize, averageRegionSize) to check is: + // 1. prevent moving small regions between stores with close scores, leading to unnecessary balance. + // 2. prevent moving huge regions, leading to over balance. + sourceID := p.source.GetID() + targetID := p.target.GetID() // Make sure after move, source score is still greater than target score. shouldBalance := p.sourceScore > p.targetScore @@ -124,24 +150,25 @@ func (p *solver) shouldBalance(scheduleName string) bool { log.Debug("skip balance "+p.kind.Resource.String(), zap.String("scheduler", scheduleName), zap.Uint64("region-id", p.region.GetID()), zap.Uint64("source-store", sourceID), zap.Uint64("target-store", targetID), zap.Int64("source-size", p.source.GetRegionSize()), zap.Float64("source-score", p.sourceScore), - zap.Int64("source-influence", sourceInfluence), zap.Int64("target-size", p.target.GetRegionSize()), zap.Float64("target-score", p.targetScore), - zap.Int64("target-influence", targetInfluence), zap.Int64("average-region-size", p.GetAverageRegionSize()), - zap.Int64("tolerant-resource", tolerantResource)) + zap.Int64("tolerant-resource", p.getTolerantResource())) } return shouldBalance } func (p *solver) getTolerantResource() int64 { - if p.kind.Resource == core.LeaderKind && p.kind.Policy == core.ByCount { - return int64(p.tolerantSizeRatio) + if p.tolerantSource > 0 { + return p.tolerantSource } - regionSize := p.region.GetApproximateSize() - if regionSize < p.GetAverageRegionSize() { - regionSize = p.GetAverageRegionSize() + + if p.kind.Resource == core.LeaderKind && p.kind.Policy == core.ByCount { + p.tolerantSource = int64(p.tolerantSizeRatio) + } else { + regionSize := p.GetAverageRegionSize() + p.tolerantSource = int64(float64(regionSize) * p.tolerantSizeRatio) } - return int64(float64(regionSize) * p.tolerantSizeRatio) + return p.tolerantSource } func adjustTolerantRatio(cluster schedule.Cluster, kind core.ScheduleKind) float64 { diff --git a/server/statistics/kind.go b/server/statistics/kind.go index f9cd78a76263..612829379552 100644 --- a/server/statistics/kind.go +++ b/server/statistics/kind.go @@ -208,6 +208,15 @@ func (rw RWType) Inverse() RWType { } } +// ForeachRegionStats foreach all region stats of read and write. +func ForeachRegionStats(f func(RWType, int, RegionStatKind)) { + for _, rwTy := range []RWType{Read, Write} { + for dim, kind := range rwTy.RegionStats() { + f(rwTy, dim, kind) + } + } +} + // GetLoadRatesFromPeer gets the load rates of the read or write type from PeerInfo. func (rw RWType) GetLoadRatesFromPeer(peer *core.PeerInfo) []float64 { deltaLoads := peer.GetLoads() diff --git a/server/statistics/store_load.go b/server/statistics/store_load.go index 0525a52c4475..a7cc723c74a4 100644 --- a/server/statistics/store_load.go +++ b/server/statistics/store_load.go @@ -144,18 +144,6 @@ func (s *StoreSummaryInfo) SetEngineAsTiFlash() { s.isTiFlash = true } -// GetPendingInfluence returns the current pending influence. -func GetPendingInfluence(stores []*core.StoreInfo) map[uint64]*Influence { - stInfos := SummaryStoreInfos(stores) - ret := make(map[uint64]*Influence, len(stInfos)) - for id, info := range stInfos { - if info.PendingSum != nil { - ret[id] = info.PendingSum - } - } - return ret -} - // StoreLoad records the current load. type StoreLoad struct { Loads []float64 diff --git a/tests/client/client_test.go b/tests/client/client_test.go index 0fa966d00716..b2d7c23adace 100644 --- a/tests/client/client_test.go +++ b/tests/client/client_test.go @@ -347,6 +347,69 @@ func TestTSOFollowerProxy(t *testing.T) { wg.Wait() } +// TestUnavailableTimeAfterLeaderIsReady is used to test https://github.com/tikv/pd/issues/5207 +func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 3) + re.NoError(err) + defer cluster.Destroy() + + endpoints := runServer(re, cluster) + cli := setupCli(re, ctx, endpoints) + + var wg sync.WaitGroup + var maxUnavailableTime, leaderReadyTime time.Time + getTsoFunc := func() { + defer wg.Done() + var lastTS uint64 + for i := 0; i < tsoRequestRound; i++ { + var physical, logical int64 + var ts uint64 + physical, logical, err = cli.GetTS(context.Background()) + ts = tsoutil.ComposeTS(physical, logical) + if err != nil { + maxUnavailableTime = time.Now() + continue + } + re.NoError(err) + re.Less(lastTS, ts) + lastTS = ts + } + } + + // test resign pd leader or stop pd leader + wg.Add(1 + 1) + go getTsoFunc() + go func() { + defer wg.Done() + leader := cluster.GetServer(cluster.GetLeader()) + leader.Stop() + cluster.WaitLeader() + leaderReadyTime = time.Now() + cluster.RunServers([]*tests.TestServer{leader}) + }() + wg.Wait() + re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) + + // test kill pd leader pod or network of leader is unreachable + wg.Add(1 + 1) + maxUnavailableTime, leaderReadyTime = time.Time{}, time.Time{} + go getTsoFunc() + go func() { + defer wg.Done() + leader := cluster.GetServer(cluster.GetLeader()) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) + leader.Stop() + cluster.WaitLeader() + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) + leaderReadyTime = time.Now() + }() + wg.Wait() + re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) +} + func TestGlobalAndLocalTSO(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/tests/client/go.mod b/tests/client/go.mod index cee2b19e8bdb..4cdc8f323e30 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad + github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a github.com/stretchr/testify v1.7.0 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 @@ -92,7 +92,7 @@ require ( github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect - github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect + github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d // indirect github.com/pingcap/tidb-dashboard v0.0.0-20220728104842-3743e533b594 // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect diff --git a/tests/client/go.sum b/tests/client/go.sum index 8d388c14cebb..4591bd2ef17f 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -390,14 +390,15 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad h1:lGKxsEwdE0pVXzHYD1SQ1vfa3t/bFVU/latrQz8b/w0= github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a h1:McYxPhA8SHqfUtLfQHHN0fQl4dy93IkhlX4Pp2MKIFA= +github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v0.0.0-20210906054005-afc726e70354/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g= github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM= github.com/pingcap/tidb-dashboard v0.0.0-20220728104842-3743e533b594 h1:kL1CW5qsn459kHZ2YoBYb+YOSWjSlshk55YP/XNQNWo= diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index e65711a5b2d1..7b36618e62ca 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -207,42 +207,12 @@ func BenchmarkDoRequestWithServiceMiddleware(b *testing.B) { resp.Body.Close() b.StartTimer() for i := 0; i < b.N; i++ { - doTestRequest(leader) + doTestRequestWithLogAudit(leader) } cancel() cluster.Destroy() } -func BenchmarkDoRequestWithoutServiceMiddleware(b *testing.B) { - b.StopTimer() - ctx, cancel := context.WithCancel(context.Background()) - cluster, _ := tests.NewTestCluster(ctx, 1) - cluster.RunInitialServers() - cluster.WaitLeader() - leader := cluster.GetServer(cluster.GetLeader()) - input := map[string]interface{}{ - "enable-audit": "false", - } - data, _ := json.Marshal(input) - req, _ := http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) - resp, _ := dialClient.Do(req) - resp.Body.Close() - b.StartTimer() - for i := 0; i < b.N; i++ { - doTestRequest(leader) - } - cancel() - cluster.Destroy() -} - -func doTestRequest(srv *tests.TestServer) { - timeUnix := time.Now().Unix() - 20 - req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", srv.GetAddr(), timeUnix), nil) - req.Header.Set("component", "test") - resp, _ := dialClient.Do(req) - resp.Body.Close() -} - func (suite *middlewareTestSuite) TestRateLimitMiddleware() { leader := suite.cluster.GetServer(suite.cluster.GetLeader()) input := map[string]interface{}{ @@ -433,7 +403,7 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { defer resp.Body.Close() content, _ := io.ReadAll(resp.Body) output := string(content) - suite.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",method=\"HTTP\",service=\"GetTrend\"} 1") + suite.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 1") // resign to test persist config oldLeaderName := leader.GetServer().Name() @@ -459,7 +429,7 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { defer resp.Body.Close() content, _ = io.ReadAll(resp.Body) output = string(content) - suite.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",method=\"HTTP\",service=\"GetTrend\"} 2") + suite.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 2") input = map[string]interface{}{ "enable-audit": "false", @@ -521,13 +491,35 @@ func BenchmarkDoRequestWithLocalLogAudit(b *testing.B) { resp.Body.Close() b.StartTimer() for i := 0; i < b.N; i++ { - doTestRequest(leader) + doTestRequestWithLogAudit(leader) } cancel() cluster.Destroy() } -func BenchmarkDoRequestWithoutLocalLogAudit(b *testing.B) { +func BenchmarkDoRequestWithPrometheusAudit(b *testing.B) { + b.StopTimer() + ctx, cancel := context.WithCancel(context.Background()) + cluster, _ := tests.NewTestCluster(ctx, 1) + cluster.RunInitialServers() + cluster.WaitLeader() + leader := cluster.GetServer(cluster.GetLeader()) + input := map[string]interface{}{ + "enable-audit": "true", + } + data, _ := json.Marshal(input) + req, _ := http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) + resp, _ := dialClient.Do(req) + resp.Body.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + doTestRequestWithPrometheus(leader) + } + cancel() + cluster.Destroy() +} + +func BenchmarkDoRequestWithoutServiceMiddleware(b *testing.B) { b.StopTimer() ctx, cancel := context.WithCancel(context.Background()) cluster, _ := tests.NewTestCluster(ctx, 1) @@ -543,12 +535,27 @@ func BenchmarkDoRequestWithoutLocalLogAudit(b *testing.B) { resp.Body.Close() b.StartTimer() for i := 0; i < b.N; i++ { - doTestRequest(leader) + doTestRequestWithLogAudit(leader) } cancel() cluster.Destroy() } +func doTestRequestWithLogAudit(srv *tests.TestServer) { + req, _ := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/pd/api/v1/admin/cache/regions", srv.GetAddr()), nil) + req.Header.Set("component", "test") + resp, _ := dialClient.Do(req) + resp.Body.Close() +} + +func doTestRequestWithPrometheus(srv *tests.TestServer) { + timeUnix := time.Now().Unix() - 20 + req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", srv.GetAddr(), timeUnix), nil) + req.Header.Set("component", "test") + resp, _ := dialClient.Do(req) + resp.Body.Close() +} + type redirectorTestSuite struct { suite.Suite cleanup func() diff --git a/tools/pd-simulator/simulator/node.go b/tools/pd-simulator/simulator/node.go index c2e904dd85df..a6a9c4787351 100644 --- a/tools/pd-simulator/simulator/node.go +++ b/tools/pd-simulator/simulator/node.go @@ -24,6 +24,7 @@ import ( "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/tools/pd-simulator/simulator/cases" "github.com/tikv/pd/tools/pd-simulator/simulator/info" "github.com/tikv/pd/tools/pd-simulator/simulator/simutil" @@ -48,7 +49,7 @@ type Node struct { ctx context.Context cancel context.CancelFunc raftEngine *RaftEngine - ioRate int64 + limiter *ratelimit.RateLimiter sizeMutex sync.Mutex } @@ -90,6 +91,8 @@ func NewNode(s *cases.Store, pdAddr string, config *SimConfig) (*Node, error) { cancel() return nil, err } + ratio := int64(time.Second) / config.SimTickInterval.Milliseconds() + speed := config.StoreIOMBPerSecond * units.MiB * ratio return &Node{ Store: store, stats: stats, @@ -98,7 +101,7 @@ func NewNode(s *cases.Store, pdAddr string, config *SimConfig) (*Node, error) { cancel: cancel, tasks: make(map[uint64]Task), receiveRegionHeartbeatCh: receiveRegionHeartbeatCh, - ioRate: config.StoreIOMBPerSecond * units.MiB, + limiter: ratelimit.NewRateLimiter(float64(speed), int(speed)), tick: uint64(rand.Intn(storeHeartBeatPeriod)), }, nil } @@ -155,7 +158,7 @@ func (n *Node) stepTask() { for _, task := range n.tasks { task.Step(n.raftEngine) if task.IsFinished() { - simutil.Logger.Debug("task finished", + simutil.Logger.Debug("task status", zap.Uint64("node-id", n.Id), zap.Uint64("region-id", task.RegionID()), zap.String("task", task.Desc())) diff --git a/tools/pd-simulator/simulator/raft.go b/tools/pd-simulator/simulator/raft.go index 76c8587431c9..cfd1e8719f03 100644 --- a/tools/pd-simulator/simulator/raft.go +++ b/tools/pd-simulator/simulator/raft.go @@ -139,7 +139,7 @@ func (r *RaftEngine) stepSplit(region *core.RegionInfo) { if r.useTiDBEncodedKey { splitKey, err = simutil.GenerateTiDBEncodedSplitKey(region.GetStartKey(), region.GetEndKey()) if err != nil { - simutil.Logger.Fatal("generate TiDB encoded split key failed", zap.Error(err)) + simutil.Logger.Fatal("Generate TiDB encoded split key failed", zap.Error(err)) } } else { splitKey = simutil.GenerateSplitKey(region.GetStartKey(), region.GetEndKey()) diff --git a/tools/pd-simulator/simulator/task.go b/tools/pd-simulator/simulator/task.go index e502b0be40d3..083d8b6774c8 100644 --- a/tools/pd-simulator/simulator/task.go +++ b/tools/pd-simulator/simulator/task.go @@ -17,7 +17,9 @@ package simulator import ( "bytes" "fmt" + "time" + "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/eraftpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" @@ -25,6 +27,28 @@ import ( "github.com/tikv/pd/tools/pd-analysis/analysis" ) +var ( + chunkSize = int64(4 * units.KiB) + maxSnapGeneratorPoolSize = uint32(2) + maxSnapReceivePoolSize = uint32(4) + compressionRatio = int64(2) +) + +type snapAction int + +const ( + generate = iota + receive +) + +type snapStatus int + +const ( + pending snapStatus = iota + running + finished +) + // Task running in node. type Task interface { Desc() string @@ -45,14 +69,8 @@ func responseToTask(resp *pdpb.RegionHeartbeatResponse, r *RaftEngine) Task { case eraftpb.ConfChangeType_AddNode: return &addPeer{ regionID: regionID, - size: region.GetApproximateSize(), - keys: region.GetApproximateKeys(), - speed: 100 * 1000 * 1000, epoch: epoch, peer: changePeer.GetPeer(), - // This two variables are used to simulate sending and receiving snapshot processes. - sendingStat: &snapshotStat{"sending", region.GetApproximateSize(), false}, - receivingStat: &snapshotStat{"receiving", region.GetApproximateSize(), false}, } case eraftpb.ConfChangeType_RemoveNode: return &removePeer{ @@ -68,9 +86,11 @@ func responseToTask(resp *pdpb.RegionHeartbeatResponse, r *RaftEngine) Task { regionID: regionID, size: region.GetApproximateSize(), keys: region.GetApproximateKeys(), - speed: 100 * 1000 * 1000, epoch: epoch, peer: changePeer.GetPeer(), + // This two variables are used to simulate sending and receiving snapshot processes. + sendingStat: newSnapshotState(region.GetApproximateSize(), generate), + receivingStat: newSnapshotState(region.GetApproximateSize(), receive), } } } else if resp.GetTransferLeader() != nil { @@ -94,9 +114,22 @@ func responseToTask(resp *pdpb.RegionHeartbeatResponse, r *RaftEngine) Task { } type snapshotStat struct { - kind string + action snapAction remainSize int64 - finished bool + status snapStatus + start time.Time +} + +func newSnapshotState(size int64, action snapAction) *snapshotStat { + if action == receive { + size /= compressionRatio + } + return &snapshotStat{ + remainSize: size, + action: action, + status: pending, + start: time.Now(), + } } type mergeRegion struct { @@ -209,15 +242,10 @@ func (t *transferLeader) IsFinished() bool { } type addPeer struct { - regionID uint64 - size int64 - keys int64 - speed int64 - epoch *metapb.RegionEpoch - peer *metapb.Peer - finished bool - sendingStat *snapshotStat - receivingStat *snapshotStat + regionID uint64 + epoch *metapb.RegionEpoch + peer *metapb.Peer + finished bool } func (a *addPeer) Desc() string { @@ -234,44 +262,19 @@ func (a *addPeer) Step(r *RaftEngine) { return } - snapshotSize := region.GetApproximateSize() - sendNode := r.conn.Nodes[region.GetLeader().GetStoreId()] - if sendNode == nil { - a.finished = true - return - } - if !processSnapshot(sendNode, a.sendingStat, snapshotSize) { - return - } - r.schedulerStats.snapshotStats.incSendSnapshot(sendNode.Id) - - recvNode := r.conn.Nodes[a.peer.GetStoreId()] - if recvNode == nil { - a.finished = true - return - } - if !processSnapshot(recvNode, a.receivingStat, snapshotSize) { - return - } - r.schedulerStats.snapshotStats.incReceiveSnapshot(recvNode.Id) - - a.size -= a.speed - if a.size < 0 { - var opts []core.RegionCreateOption - if region.GetPeer(a.peer.GetId()) == nil { - opts = append(opts, core.WithAddPeer(a.peer)) - r.schedulerStats.taskStats.incAddPeer(region.GetID()) - } else { - opts = append(opts, core.WithPromoteLearner(a.peer.GetId())) - r.schedulerStats.taskStats.incPromoteLeaner(region.GetID()) - } - opts = append(opts, core.WithIncConfVer()) - newRegion := region.Clone(opts...) - r.SetRegion(newRegion) - r.recordRegionChange(newRegion) - recvNode.incUsedSize(uint64(snapshotSize)) - a.finished = true + var opts []core.RegionCreateOption + if region.GetPeer(a.peer.GetId()) == nil { + opts = append(opts, core.WithAddPeer(a.peer)) + r.schedulerStats.taskStats.incAddPeer(region.GetID()) + } else { + opts = append(opts, core.WithPromoteLearner(a.peer.GetId())) + r.schedulerStats.taskStats.incPromoteLeaner(region.GetID()) } + opts = append(opts, core.WithIncConfVer()) + newRegion := region.Clone(opts...) + r.SetRegion(newRegion) + r.recordRegionChange(newRegion) + a.finished = true } func (a *addPeer) RegionID() uint64 { @@ -352,13 +355,14 @@ func (a *removePeer) IsFinished() bool { } type addLearner struct { - regionID uint64 - size int64 - keys int64 - speed int64 - epoch *metapb.RegionEpoch - peer *metapb.Peer - finished bool + regionID uint64 + size int64 + keys int64 + epoch *metapb.RegionEpoch + peer *metapb.Peer + finished bool + sendingStat *snapshotStat + receivingStat *snapshotStat } func (a *addLearner) Desc() string { @@ -375,21 +379,41 @@ func (a *addLearner) Step(r *RaftEngine) { return } - a.size -= a.speed - if a.size < 0 { - if region.GetPeer(a.peer.GetId()) == nil { - newRegion := region.Clone( - core.WithAddPeer(a.peer), - core.WithIncConfVer(), - ) - r.SetRegion(newRegion) - r.recordRegionChange(newRegion) - r.schedulerStats.taskStats.incAddLeaner(region.GetID()) - } + snapshotSize := region.GetApproximateSize() + sendNode := r.conn.Nodes[region.GetLeader().GetStoreId()] + if sendNode == nil { a.finished = true - if analysis.GetTransferCounter().IsValid { - analysis.GetTransferCounter().AddTarget(a.regionID, a.peer.StoreId) - } + return + } + if !processSnapshot(sendNode, a.sendingStat) { + return + } + r.schedulerStats.snapshotStats.incSendSnapshot(sendNode.Id) + + recvNode := r.conn.Nodes[a.peer.GetStoreId()] + if recvNode == nil { + a.finished = true + return + } + if !processSnapshot(recvNode, a.receivingStat) { + return + } + r.schedulerStats.snapshotStats.incReceiveSnapshot(recvNode.Id) + + if region.GetPeer(a.peer.GetId()) == nil { + newRegion := region.Clone( + core.WithAddPeer(a.peer), + core.WithIncConfVer(), + ) + r.SetRegion(newRegion) + r.recordRegionChange(newRegion) + r.schedulerStats.taskStats.incAddLeaner(region.GetID()) + recvNode.incUsedSize(uint64(snapshotSize)) + a.finished = true + } + + if analysis.GetTransferCounter().IsValid { + analysis.GetTransferCounter().AddTarget(a.regionID, a.peer.StoreId) } } @@ -401,23 +425,39 @@ func (a *addLearner) IsFinished() bool { return a.finished } -func processSnapshot(n *Node, stat *snapshotStat, snapshotSize int64) bool { - // If the statement is true, it will start to send or receive the snapshot. - if stat.remainSize == snapshotSize { - if stat.kind == "sending" { +func processSnapshot(n *Node, stat *snapshotStat) bool { + if stat.status == finished { + return true + } + if stat.status == pending { + if stat.action == generate && n.stats.SendingSnapCount > maxSnapGeneratorPoolSize { + return false + } + if stat.action == receive && n.stats.ReceivingSnapCount > maxSnapReceivePoolSize { + return false + } + stat.status = running + // If the statement is true, it will start to send or Receive the snapshot. + if stat.action == generate { n.stats.SendingSnapCount++ } else { n.stats.ReceivingSnapCount++ } } - stat.remainSize -= n.ioRate - // The sending or receiving process has not finished yet. + + // store should Generate/Receive snapshot by chunk size. + // todo: the process of snapshot is single thread, the later snapshot task must wait the first one. + for n.limiter.AllowN(int(chunkSize)) { + stat.remainSize -= chunkSize + } + + // The sending or receiving process has not status yet. if stat.remainSize > 0 { return false } - if !stat.finished { - stat.finished = true - if stat.kind == "sending" { + if stat.status == running { + stat.status = finished + if stat.action == generate { n.stats.SendingSnapCount-- } else { n.stats.ReceivingSnapCount-- diff --git a/tools/pd-tso-bench/go.mod b/tools/pd-tso-bench/go.mod index 452a6d8a8424..5864ea5d9dee 100644 --- a/tools/pd-tso-bench/go.mod +++ b/tools/pd-tso-bench/go.mod @@ -5,7 +5,7 @@ go 1.16 require ( github.com/influxdata/tdigest v0.0.1 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c - github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee + github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/prometheus/client_golang v1.11.0 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.uber.org/zap v1.20.0 diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index 4e22744e5acf..068c3a13b5b4 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -106,8 +106,8 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad h1:lGKxsEwdE0pVXzHYD1SQ1vfa3t/bFVU/latrQz8b/w0= github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=