-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
kv: introduce a rate limiter for the range consistency checker
Closes #47290 This commit introduces a new flag server.consistency_check.max_rate to control the rate at which the consistency checker may scan through the range to compute it's checksum. Without a rate limit in place the checker cmay overwhelm the cluster with sufficiently large nodes. For example on a 10B node the checker is expected to produce 120MB/second of disk reads. The flag is defined as bytes/second and set to 8MB/s by default. We expect the customers to continue to use it in conjunction with server.consistency_check.interval, which will givem the ability to control both the frequency and speed of checks. Release note (performance improvement): Introduce a new flag server.consistency_check.max_rate expressed in bytes/second to throttle the rate at which cockroach scans through the disk to perform a consistency check. This control is necessary to ensure smooth performance on a cluster with large node sizes (i.e. in the 10TB+ range)
- Loading branch information
Showing
12 changed files
with
156 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| // Copyright 2020 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package limit | ||
|
|
||
| import ( | ||
| "context" | ||
| "math/big" | ||
|
|
||
| "golang.org/x/time/rate" | ||
| ) | ||
|
|
||
| // maxInt is the maximum int allowed by the architecture running this code i.e. | ||
| // it could be int32 or int64, unfortunately golang does not have a built in | ||
| // field for this. | ||
| const maxInt = int(^uint(0) >> 1) | ||
|
|
||
| // LimiterBurstDisabled is used to solve a complication in rate.Limiter. | ||
| // The rate.Limiter requires a burst parameter and if the throttled value | ||
| // exceeds the burst it just fails. This not always the desired behavior, | ||
| // sometimes we want the limiter to apply the throttle and not enforce any | ||
| // hard limits on an arbitrarily large value. This feature is particularly | ||
| // useful in Cockroach, when we want to throttle on the KV pair, the size | ||
| // of which is not strictly enforced. | ||
| type LimiterBurstDisabled struct { | ||
| // Avoid embedding, as most methods on the limiter take the parameter | ||
| // burst into consideration. | ||
| limiter *rate.Limiter | ||
| } | ||
|
|
||
| // NewLimiter returns a new LimiterBurstDisabled that allows events up to rate r. | ||
| func NewLimiter(r rate.Limit) *LimiterBurstDisabled { | ||
| // Unfortunately we can't disable the burst parameter on the | ||
| // limiter, so we have to provide some value to it. To remove the cognitive | ||
| // burden from the user, we set this value to be equal to the limit. | ||
| // Semantically the choice of burst parameter does not matter, since | ||
| // we will loop the limiter until all the tokens have been consumed. However | ||
| // we want to minimize the number of loops for performance, which is why | ||
| // setting the burst parameter to the limit is a good trade off. | ||
| var burst, _ = big.NewFloat(float64(r)).Int64() | ||
| if burst > int64(maxInt) { | ||
| burst = int64(maxInt) | ||
| } | ||
| return &LimiterBurstDisabled{ | ||
| limiter: rate.NewLimiter(r, int(burst)), | ||
| } | ||
| } | ||
|
|
||
| // WaitN blocks until lim permits n events to happen. | ||
| // | ||
| // This function will now only return an error if the Context is canceled and | ||
| // should never in practice hit the burst check in the underlying limiter. | ||
| func (lim *LimiterBurstDisabled) WaitN(ctx context.Context, n int) error { | ||
| for n > 0 { | ||
| cur := n | ||
| if cur > lim.limiter.Burst() { | ||
| cur = lim.limiter.Burst() | ||
| } | ||
| if err := lim.limiter.WaitN(ctx, cur); err != nil { | ||
| return err | ||
| } | ||
| n -= cur | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| // Copyright 2020 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package limit | ||
|
|
||
| import ( | ||
| "context" | ||
| "math" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "golang.org/x/time/rate" | ||
| ) | ||
|
|
||
| func TestLimiterBurstDisabled(t *testing.T) { | ||
| limiter := NewLimiter(100) | ||
|
|
||
| if err := limiter.WaitN(context.Background(), 101); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| limiter = NewLimiter(rate.Limit(math.MaxFloat64)) | ||
| require.Equal(t, maxInt, limiter.limiter.Burst()) | ||
| } |