From 1f77f7f8329d658464400c6dcefbb1b00e92225c Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Mon, 25 Sep 2023 22:40:59 -0400 Subject: [PATCH] rpc: permit gRPC stream window sizes up to 64 MB CockroachDB accepts a COCKROACH_RPC_INITIAL_WINDOW_SIZE environment variable that can be used to set the initial window size for gRPC streams. However, the setting could previously only be used to reduce the initial window size, not to increase it. This commit fixes that, allowing the initial window size to be increased up to 64 MB. This is useful for multi-region clusters that want to push high volumes of writes over high-latency WAN links. Release note (general change): Increased the maximum permitted value of the COCKROACH_RPC_INITIAL_WINDOW_SIZE environment variable to 64MB. When used in conjunction with a tuned OS-level maximum TCP window size, this can increase the throughput that Raft replication can sustain over high latency network links. --- pkg/rpc/settings.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/rpc/settings.go b/pkg/rpc/settings.go index 2ba24af95e90..31ae22dfcd29 100644 --- a/pkg/rpc/settings.go +++ b/pkg/rpc/settings.go @@ -52,11 +52,10 @@ var useDialback = settings.RegisterBoolSetting( var enableRPCCompression = envutil.EnvOrDefaultBool("COCKROACH_ENABLE_RPC_COMPRESSION", true) func getWindowSize(ctx context.Context, name string, c ConnectionClass, defaultSize int) int32 { - const maxWindowSize = defaultWindowSize * 32 s := envutil.EnvOrDefaultInt(name, defaultSize) - if s > maxWindowSize { - log.Warningf(ctx, "%s value too large; trimmed to %d", name, maxWindowSize) - s = maxWindowSize + if s > maximumWindowSize { + log.Warningf(ctx, "%s value too large; trimmed to %d", name, maximumWindowSize) + s = maximumWindowSize } if s <= defaultWindowSize { log.Warningf(ctx, @@ -66,7 +65,9 @@ func getWindowSize(ctx context.Context, name string, c ConnectionClass, defaultS } const ( - defaultWindowSize = 65535 + defaultWindowSize = 65535 // from gRPC + defaultInitialWindowSize = defaultWindowSize * 32 // 2MB + maximumWindowSize = defaultInitialWindowSize * 32 // 64MB // The coefficient by which the tolerated offset is multiplied to determine // the maximum acceptable measurement latency. maximumPingDurationMult = 2 @@ -76,7 +77,7 @@ const ( type windowSizeSettings struct { values struct { init sync.Once - // initialWindowSize is the initial window size for a connection. + // initialWindowSize is the initial window size for a gRPC stream. initialWindowSize int32 // initialConnWindowSize is the initial window size for a connection. initialConnWindowSize int32 @@ -88,10 +89,13 @@ type windowSizeSettings struct { func (s *windowSizeSettings) maybeInit(ctx context.Context) { s.values.init.Do(func() { s.values.initialWindowSize = getWindowSize(ctx, - "COCKROACH_RPC_INITIAL_WINDOW_SIZE", DefaultClass, defaultWindowSize*32) + "COCKROACH_RPC_INITIAL_WINDOW_SIZE", DefaultClass, defaultInitialWindowSize) s.values.initialConnWindowSize = s.values.initialWindowSize * 16 + if s.values.initialConnWindowSize > maximumWindowSize { + s.values.initialConnWindowSize = maximumWindowSize + } s.values.rangefeedInitialWindowSize = getWindowSize(ctx, - "COCKROACH_RANGEFEED_RPC_INITIAL_WINDOW_SIZE", RangefeedClass, 2*defaultWindowSize /* 128K */) + "COCKROACH_RANGEFEED_RPC_INITIAL_WINDOW_SIZE", RangefeedClass, 2*defaultWindowSize /* 128KB */) }) }