From defd0dee0a69c6d244f099e12e8aa8b96beb6cbc Mon Sep 17 00:00:00 2001 From: Oleg Afanasyev Date: Thu, 24 Jun 2021 19:24:48 +0100 Subject: [PATCH] kvcoord: increase max_intent_bytes default to 4MB Previously limit was set conservatively to 256KB. Having a conservative limit can cause intent cleanup batches to condense aggressively resorting to range cleanups which are more expensive. Increasing defaults will reduce contention during cleanup at the expense of increased memory for large transactions on tx_coord_sender. Release note (performance improvement): Increase default value for kv.transaction.max_intents_bytes cluster setting from 256KB to 4MB to improve transaction performance for large transactions at the expense of increased memory usage. Transactions above this size limit use slower cleanup mode for commits and aborts. --- docs/generated/settings/settings-for-tenants.txt | 2 +- docs/generated/settings/settings.html | 2 +- pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt index e2cc1ecf6df4..af6a1f726d39 100644 --- a/docs/generated/settings/settings-for-tenants.txt +++ b/docs/generated/settings/settings-for-tenants.txt @@ -32,7 +32,7 @@ kv.range_split.by_load_enabled boolean true allow automatic splits of ranges bas kv.range_split.load_qps_threshold integer 2500 the QPS over which, the range becomes a candidate for load based splitting kv.rangefeed.enabled boolean false if set, rangefeed registration is enabled kv.replication_reports.interval duration 1m0s the frequency for generating the replication_constraint_stats, replication_stats_report and replication_critical_localities reports (set to 0 to disable) -kv.transaction.max_intents_bytes integer 262144 maximum number of bytes used to track locks in transactions +kv.transaction.max_intents_bytes integer 4194304 maximum number of bytes used to track locks in transactions kv.transaction.max_refresh_spans_bytes integer 256000 maximum number of bytes used to track refresh spans in serializable transactions security.ocsp.mode enumeration off "use OCSP to check whether TLS certificates are revoked. If the OCSP server is unreachable, in strict mode all certificates will be rejected diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index 71965d154498..73d6327e8d7d 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -36,7 +36,7 @@ kv.replication_reports.intervalduration1m0sthe frequency for generating the replication_constraint_stats, replication_stats_report and replication_critical_localities reports (set to 0 to disable) kv.snapshot_rebalance.max_ratebyte size8.0 MiBthe rate limit (bytes/sec) to use for rebalance and upreplication snapshots kv.snapshot_recovery.max_ratebyte size8.0 MiBthe rate limit (bytes/sec) to use for recovery snapshots -kv.transaction.max_intents_bytesinteger262144maximum number of bytes used to track locks in transactions +kv.transaction.max_intents_bytesinteger4194304maximum number of bytes used to track locks in transactions kv.transaction.max_refresh_spans_bytesinteger256000maximum number of bytes used to track refresh spans in serializable transactions security.ocsp.modeenumerationoffuse OCSP to check whether TLS certificates are revoked. If the OCSP
server is unreachable, in strict mode all certificates will be rejected
and in lax mode all certificates will be accepted. [off = 0, lax = 1, strict = 2] security.ocsp.timeoutduration3stimeout before considering the OCSP server unreachable diff --git a/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go b/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go index a3a4e04fc4bf..cd3ffb283872 100644 --- a/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go +++ b/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go @@ -67,10 +67,17 @@ var pipelinedWritesMaxBatchSize = settings.RegisterIntSetting( // NB: this is called "max_intents_bytes" instead of "max_lock_bytes" because // it was created before the concept of intents were generalized to locks. // Switching it would require a migration which doesn't seem worth it. +// +// Note: Default value was arbitrarily set to 256KB but practice showed that +// it could be raised higher. When transaction reaches this limit, intent +// cleanup switches to range instead of point cleanup which is expensive +// because it needs to hold broad latches and iterate through the range to +// find matching intents. +// See #54029 for more details. var trackedWritesMaxSize = settings.RegisterIntSetting( "kv.transaction.max_intents_bytes", "maximum number of bytes used to track locks in transactions", - 1<<18, /* 256 KB */ + 1<<22, /* 4 MB */ ).WithPublic() // txnPipeliner is a txnInterceptor that pipelines transactional writes by using