forked from cockroachdb/cockroach
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
kvcoord: setting to reject txns above lock span limit
This patch introduces kv.transaction.reject_over_max_intents_budget. If set, this changes our behavior when a txn exceeds its locks+in-flight write budget (kv.transaction.max_intents_bytes): instead of compacting some of its lock spans with precision loss, the request causing the budget to be exceeded will be rejected instead. The idea is that we've seen transactions that exceed this budget be very expensive to clean up - they have to scan a lot to find their intents, and these cleanups take wide latches. So now one has the option to reject these transactions, instead of risking this performance cliff. Each request is checked against the budget by the pipeliner before being sent out for evaluation. This check is not precise, since the exact effects of the request on the memory budget are only known at response time because of ResumeSpans, effects of QueryIntents, etc. So, the check is best-effort. If a slips through and then the response overflows the budget, we keep the locks non-condensed; if a further request in the txn tries to lock more, it'll be rejected. A commit/rollback is always allowed to pass through, since it doesn't lock anything by itself. Fixes cockroachdb#66742 Release note (general change): A new cluster setting (kv.transaction.reject_over_max_intents_budget) affords control over the behavior when a transaction exceeds its "locks-tracking memory budget" (dictated by kv.transaction.max_intents_bytes). Instead of allowing such transaction to continue with imprecise tracking of their locks, setting this new option rejects the query that would push its transaction over this budget with an error (error code 53400 - "configuration limit exceeded). Transactions that don't track their locks precisely are potentially destabilizing for the cluster since cleaning them up can take considerable resources. Transactions that change many rows have the potential to run into this memory budget issue.
- Loading branch information
1 parent
94ac161
commit 47ecb43
Showing
12 changed files
with
580 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| // Copyright 2021 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package kvcoord | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/roachpb" | ||
| "github.com/cockroachdb/errors" | ||
| "github.com/cockroachdb/errors/errorspb" | ||
| "github.com/gogo/protobuf/proto" | ||
| ) | ||
|
|
||
| // lockSpansOverBudgetError signals that a txn is being rejected because lock | ||
| // spans do not fit in their memory budget. | ||
| type lockSpansOverBudgetError struct { | ||
| lockSpansBytes int64 | ||
| limitBytes int64 | ||
| baSummary string | ||
| txnDetails string | ||
| } | ||
|
|
||
| func newLockSpansOverBudgetError( | ||
| lockSpansBytes, limitBytes int64, ba roachpb.BatchRequest, | ||
| ) lockSpansOverBudgetError { | ||
| return lockSpansOverBudgetError{ | ||
| lockSpansBytes: lockSpansBytes, | ||
| limitBytes: limitBytes, | ||
| baSummary: ba.Summary(), | ||
| txnDetails: ba.Txn.String(), | ||
| } | ||
| } | ||
|
|
||
| func (l lockSpansOverBudgetError) Error() string { | ||
| return fmt.Sprintf("the transaction is locking too many rows and exceeded its lock-tracking memory budget; "+ | ||
| "lock spans: %d bytes > budget: %d bytes. Request pushing transaction over the edge: %s. "+ | ||
| "Transaction details: %s.", l.lockSpansBytes, l.limitBytes, l.baSummary, l.txnDetails) | ||
| } | ||
|
|
||
| func encodeLockSpansOverBudgetError( | ||
| _ context.Context, err error, | ||
| ) (msgPrefix string, safe []string, details proto.Message) { | ||
| t := err.(lockSpansOverBudgetError) | ||
| details = &errorspb.StringsPayload{ | ||
| Details: []string{ | ||
| strconv.FormatInt(t.lockSpansBytes, 10), strconv.FormatInt(t.limitBytes, 10), | ||
| t.baSummary, t.txnDetails, | ||
| }, | ||
| } | ||
| msgPrefix = "the transaction is locking too many rows" | ||
| return msgPrefix, nil, details | ||
| } | ||
|
|
||
| func decodeLockSpansOverBudgetError( | ||
| _ context.Context, msgPrefix string, safeDetails []string, payload proto.Message, | ||
| ) error { | ||
| m, ok := payload.(*errorspb.StringsPayload) | ||
| if !ok || len(m.Details) < 4 { | ||
| // If this ever happens, this means some version of the library | ||
| // (presumably future) changed the payload type, and we're | ||
| // receiving this here. In this case, give up and let | ||
| // DecodeError use the opaque type. | ||
| return nil | ||
| } | ||
| lockBytes, decodeErr := strconv.ParseInt(m.Details[0], 10, 64) | ||
| if decodeErr != nil { | ||
| return nil //nolint:returnerrcheck | ||
| } | ||
| limitBytes, decodeErr := strconv.ParseInt(m.Details[1], 10, 64) | ||
| if decodeErr != nil { | ||
| return nil //nolint:returnerrcheck | ||
| } | ||
| return lockSpansOverBudgetError{ | ||
| lockSpansBytes: lockBytes, | ||
| limitBytes: limitBytes, | ||
| baSummary: m.Details[2], | ||
| txnDetails: m.Details[3], | ||
| } | ||
| } | ||
|
|
||
| func init() { | ||
| pKey := errors.GetTypeKey(lockSpansOverBudgetError{}) | ||
| errors.RegisterLeafEncoder(pKey, encodeLockSpansOverBudgetError) | ||
| errors.RegisterLeafDecoder(pKey, decodeLockSpansOverBudgetError) | ||
| } |
Oops, something went wrong.