base: add DialTimeout

This patch adds a `DialTimeout` constant, set to `2 * NetworkTimeout` to
account for the additional roundtrips in TCP + TLS handshakes.

Release note: None

pkg/base/config.go

@@ -118,6 +118,11 @@ func DefaultHistogramWindowInterval() time.Duration {
 }
 
 var (
+	// DialTimeout is the timeout used when dialing nodes. For gRPC, this is 3
+	// roundtrips (TCP + TLS handshake), so we set it to twice the network
+	// timeout.
+	DialTimeout = 2 * NetworkTimeout
+
 	// defaultRaftElectionTimeoutTicks specifies the number of Raft Tick
 	// invocations that must pass between elections.
 	defaultRaftElectionTimeoutTicks = envutil.EnvOrDefaultInt(

pkg/rpc/addjoin.go

@@ -16,6 +16,7 @@ import (
 	"math"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/base"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/credentials"
@@ -38,7 +39,7 @@ func GetAddJoinDialOptions(certPool *x509.CertPool) []grpc.DialOption {
 	backoffConfig.MaxDelay = time.Second
 	dialOpts = append(dialOpts, grpc.WithConnectParams(grpc.ConnectParams{
 		Backoff:           backoffConfig,
-		MinConnectTimeout: minConnectionTimeout}))
+		MinConnectTimeout: base.DialTimeout}))
 	dialOpts = append(dialOpts, grpc.WithKeepaliveParams(clientKeepalive))
 	dialOpts = append(dialOpts,
 		grpc.WithInitialWindowSize(initialWindowSize),

pkg/rpc/context.go

@@ -101,9 +101,6 @@ var (
 		"COCKROACH_RANGEFEED_RPC_INITIAL_WINDOW_SIZE", RangefeedClass, 2*defaultWindowSize /* 128K */)
 )
 
-// GRPC Dialer connection timeout.
-var minConnectionTimeout = 5 * time.Second
-
 // errDialRejected is returned from client interceptors when the server's
 // stopper is quiescing. The error is constructed to return true in
 // `grpcutil.IsConnectionRejected` which prevents infinite retry loops during
@@ -1751,18 +1748,17 @@ func (rpcCtx *Context) grpcDialRaw(
 	// our setup with onlyOnceDialer below. So note that our choice here is
 	// inconsequential assuming all works as designed.
 	backoff := time.Second
-	if backoff > minConnectionTimeout {
-		// This is for testing where we set a small minConnectionTimeout.
-		// gRPC will internally round up the min connection timeout to the max
-		// backoff. This can be unintuitive and so we opt out of it by lowering the
-		// max backoff.
-		backoff = minConnectionTimeout
+	if backoff > base.DialTimeout {
+		// This is for testing where we set a small DialTimeout. gRPC will
+		// internally round up the min connection timeout to the max backoff. This
+		// can be unintuitive and so we opt out of it by lowering the max backoff.
+		backoff = base.DialTimeout
 	}
 	backoffConfig.BaseDelay = backoff
 	backoffConfig.MaxDelay = backoff
 	dialOpts = append(dialOpts, grpc.WithConnectParams(grpc.ConnectParams{
 		Backoff:           backoffConfig,
-		MinConnectTimeout: minConnectionTimeout}))
+		MinConnectTimeout: base.DialTimeout}))
 	dialOpts = append(dialOpts, grpc.WithKeepaliveParams(clientKeepalive))
 	dialOpts = append(dialOpts, grpc.WithInitialConnWindowSize(initialConnWindowSize))
 	if class == RangefeedClass {

pkg/rpc/down_node_test.go

@@ -17,6 +17,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -33,9 +34,9 @@ func TestConnectingToDownNode(t *testing.T) {
 
 	{
 		defer func(prev time.Duration) {
-			minConnectionTimeout = prev
-		}(minConnectionTimeout)
-		minConnectionTimeout = time.Millisecond
+			base.DialTimeout = prev
+		}(base.DialTimeout)
+		base.DialTimeout = time.Millisecond
 	}
 
 	testutils.RunTrueAndFalse(t, "refused", func(t *testing.T, refused bool) {

pkg/server/pagination.go

@@ -367,7 +367,7 @@ func (r *rpcNodePaginator) queryNode(ctx context.Context, nodeID roachpb.NodeID,
 		r.mu.currentIdx++
 		r.mu.turnCond.Broadcast()
 	}
-	if err := contextutil.RunWithTimeout(ctx, "dial node", base.NetworkTimeout, func(ctx context.Context) error {
+	if err := contextutil.RunWithTimeout(ctx, "dial node", base.DialTimeout, func(ctx context.Context) error {
 		var err error
 		client, err = r.dialFn(ctx, nodeID)
 		return err

pkg/server/status.go

@@ -2681,7 +2681,7 @@ func (s *statusServer) iterateNodes(
 
 	nodeQuery := func(ctx context.Context, nodeID roachpb.NodeID) {
 		var client interface{}
-		err := contextutil.RunWithTimeout(ctx, "dial node", base.NetworkTimeout, func(ctx context.Context) error {
+		err := contextutil.RunWithTimeout(ctx, "dial node", base.DialTimeout, func(ctx context.Context) error {
 			var err error
 			client, err = dialFn(ctx, nodeID)
 			return err

pkg/server/tenant_status.go

@@ -747,7 +747,7 @@ func (t *tenantStatusServer) iteratePods(
 
 	instanceQuery := func(ctx context.Context, instance sqlinstance.InstanceInfo) {
 		var client interface{}
-		err := contextutil.RunWithTimeout(ctx, "dial instance", base.NetworkTimeout, func(ctx context.Context) error {
+		err := contextutil.RunWithTimeout(ctx, "dial instance", base.DialTimeout, func(ctx context.Context) error {
 			var err error
 			client, err = dialFn(ctx, instance.InstanceID, instance.InstanceAddr)
 			return err