From 1b2749eab6f9f00d118023e42b085950a1fb8845 Mon Sep 17 00:00:00 2001 From: sumeerbhola Date: Wed, 10 Mar 2021 18:26:56 -0500 Subject: [PATCH] storage: increase maxSize of key-value to 512MB, in pebbleMVCCScanner This is to work around the fact that the only enforcement that limits writing large key-values is based on the cluster setting kv.raft.command.max_size. So if someone increases the setting and allows a large value to be written, the read path will go into a panic. Release note: None --- pkg/storage/pebble_mvcc_scanner.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/storage/pebble_mvcc_scanner.go b/pkg/storage/pebble_mvcc_scanner.go index f07e92abd281..07c59d3395a2 100644 --- a/pkg/storage/pebble_mvcc_scanner.go +++ b/pkg/storage/pebble_mvcc_scanner.go @@ -48,7 +48,12 @@ func (p *pebbleResults) put(key MVCCKey, value []byte) { // Key value lengths take up 8 bytes (2 x Uint32). const kvLenSize = 8 const minSize = 16 - const maxSize = 128 << 20 // 128 MB + // This used to be 128MB, but the enforcement of key+value size limit is + // currently done based on the cluster setting kv.raft.command.max_size, + // which defaults to 64MB but can be increased. Until we add proper + // enforcement, we are increasing this to 512MB, under the assumption that + // no one will increase kv.raft.command.max_size to > 512MB. + const maxSize = 512 << 20 // 512 MB // We maintain a list of buffers, always encoding into the last one (a.k.a. // pebbleResults.repr). The size of the buffers is exponentially increasing,