From acb8c675060554b222ee84f2176f94412d552b37 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 14 Nov 2022 23:38:07 +0000 Subject: [PATCH] base: reduce network timeouts This patch reduces the network timeout from 3 seconds to 2 seconds. This change also affects gRPC keepalive intervals/timeouts. Furthermore, the RPC heartbeat interval is now reduced to half of the network timeout (from 3 seconds to 1 second), with a timeout equal to the network timeout (from 6 seconds to 2 seconds). The gRPC dial timeout is also reduced to the network timeout (from 5 seconds to 2 seconds). When a peer is unresponsive, these timeouts determine how quickly RPC calls (and thus critical operations such as lease acquisitions) will be retried against a different node. Reducing them therefore improves recovery time during infrastructure outages. An environment variable `COCKROACH_NETWORK_TIMEOUT` has been introduced to tweak this timeout if needed. Release note (ops change): The network timeout for RPC connections between cluster nodes has been reduced from 3 seconds to 2 seconds, in order to reduce unavailability and tail latencies during infrastructure outages. This can now be changed via the environment variable `COCKROACH_NETWORK_TIMEOUT` which is set to `2s`. --- pkg/base/config.go | 33 ++++++++++++++++++++++++++------- pkg/rpc/context.go | 2 +- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pkg/base/config.go b/pkg/base/config.go index 485b82a96f6c..1a9c3ed15ede 100644 --- a/pkg/base/config.go +++ b/pkg/base/config.go @@ -47,9 +47,6 @@ const ( defaultSQLAddr = ":" + DefaultPort defaultHTTPAddr = ":" + DefaultHTTPPort - // NetworkTimeout is the timeout used for network operations. - NetworkTimeout = 3 * time.Second - // defaultRaftTickInterval is the default resolution of the Raft timer. defaultRaftTickInterval = 200 * time.Millisecond @@ -66,10 +63,6 @@ const ( // each heartbeat. defaultRaftHeartbeatIntervalTicks = 5 - // defaultRPCHeartbeatInterval is the default value of RPCHeartbeatIntervalAndHalfTimeout - // used by the rpc context. - defaultRPCHeartbeatInterval = 3 * time.Second - // defaultRangeLeaseRenewalFraction specifies what fraction the range lease // renewal duration should be of the range lease active time. For example, // with a value of 0.2 and a lease duration of 10 seconds, leases would be @@ -118,6 +111,32 @@ func DefaultHistogramWindowInterval() time.Duration { } var ( + // NetworkTimeout is the timeout used for network operations. + // + // The maximum RTT between cloud regions is roughly 350ms both in GCP + // (asia-south2 to southamerica-west1) and AWS (af-south-1 to sa-east-1). It + // can occasionally be up to 500ms, but 400ms is a reasonable upper bound + // under nominal conditions. + // https://datastudio.google.com/reporting/fc733b10-9744-4a72-a502-92290f608571/page/70YCB + // https://www.cloudping.co/grid/p_99/timeframe/1W + // + // Linux has an RTT-dependant retransmission timeout (RTO) which we can + // approximate as 1.5x RTT (smoothed RTT + 4x RTT variance), with a lower + // bound of 200ms, so the worst-case RTO is 750ms. A round trip can thus + // take 1.25s if a single packet is lost (750ms retransmit + 500ms RTT). + // + // Initial connection attempts can take 3 RTTs (TCP + TLS). On a high-latency + // link with 500ms RTT, a single lost packet will thus cause the connection to + // fail (1.5s + 750ms > 2s), but it will succeed under nominal high latencies + // of 400ms RTT (1.2s + 600ms < 2s). Failed connections will also be retried. + NetworkTimeout = envutil.EnvOrDefaultDuration("COCKROACH_NETWORK_TIMEOUT", 2*time.Second) + + // defaultRPCHeartbeatInterval is the default value of + // RPCHeartbeatIntervalAndHalfTimeout used by the RPC context. The heartbeat + // timeout is twice this value, and we want that to be equivalent to + // NetworkTimeout to quickly detect peer unavailability. + defaultRPCHeartbeatInterval = NetworkTimeout / 2 + // defaultRaftElectionTimeoutTicks specifies the number of Raft Tick // invocations that must pass between elections. defaultRaftElectionTimeoutTicks = envutil.EnvOrDefaultInt( diff --git a/pkg/rpc/context.go b/pkg/rpc/context.go index d5b24eca5125..1f7c53f93e5b 100644 --- a/pkg/rpc/context.go +++ b/pkg/rpc/context.go @@ -102,7 +102,7 @@ var ( ) // GRPC Dialer connection timeout. -var minConnectionTimeout = 5 * time.Second +var minConnectionTimeout = base.NetworkTimeout // errDialRejected is returned from client interceptors when the server's // stopper is quiescing. The error is constructed to return true in