fix(spanner): do not mutate incoming options in createGCPMultiEndpoint

spanner/client.go

@@ -31,6 +31,7 @@ import (
 	sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
 	"github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp"
 	grpcgcppb "github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp/grpc_gcp"
+	"github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp/multiendpoint"
 	"github.com/googleapis/gax-go/v2"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
@@ -46,6 +47,7 @@ import (
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 
 	vkit "cloud.google.com/go/spanner/apiv1"
 	"cloud.google.com/go/spanner/internal"
@@ -133,7 +135,35 @@ func (c *Client) ClientID() string {
 	return c.sc.id
 }
 
-func createGCPMultiEndpoint(cfg *grpcgcp.GCPMultiEndpointOptions, config ClientConfig, opts ...option.ClientOption) (*grpcgcp.GCPMultiEndpoint, error) {
+func copyGCPMultiEndpointConfig(cfg *grpcgcp.GCPMultiEndpointOptions) *grpcgcp.GCPMultiEndpointOptions {
+	if cfg == nil {
+		return nil
+	}
+
+	config := grpcgcp.GCPMultiEndpointOptions{
+		GRPCgcpConfig:  proto.Clone(cfg.GRPCgcpConfig).(*grpcgcppb.ApiConfig),
+		MultiEndpoints: make(map[string]*multiendpoint.MultiEndpointOptions),
+		Default:        cfg.Default,
+		DialFunc:       cfg.DialFunc,
+	}
+
+	for k, v := range cfg.MultiEndpoints {
+		config.MultiEndpoints[k] = &multiendpoint.MultiEndpointOptions{
+			Endpoints:       make([]string, len(v.Endpoints)),
+			RecoveryTimeout: v.RecoveryTimeout,
+			SwitchingDelay:  v.SwitchingDelay,
+		}
+		copy(config.MultiEndpoints[k].Endpoints, v.Endpoints)
+	}
+
+	return &config
+}
+
+func createGCPMultiEndpoint(meConfig *grpcgcp.GCPMultiEndpointOptions, config ClientConfig, opts ...option.ClientOption) (*grpcgcp.GCPMultiEndpoint, error) {
+	cfg := copyGCPMultiEndpointConfig(meConfig)
+	options := make([]option.ClientOption, len(opts))
+	copy(options, opts)
+
 	if cfg.GRPCgcpConfig == nil {
 		cfg.GRPCgcpConfig = &grpcgcppb.ApiConfig{}
 	}
@@ -196,7 +226,7 @@ func createGCPMultiEndpoint(cfg *grpcgcp.GCPMultiEndpointOptions, config ClientC
 			option.WithoutAuthentication(),
 			internaloption.SkipDialSettingsValidation(),
 		}
-		opts = append(opts, emulatorOpts...)
+		options = append(options, emulatorOpts...)
 		// Replace all endpoints with emulator target.
 		for _, meo := range cfg.MultiEndpoints {
 			meo.Endpoints = []string{emulatorAddr}
@@ -216,7 +246,8 @@ func createGCPMultiEndpoint(cfg *grpcgcp.GCPMultiEndpointOptions, config ClientC
 	cfg.GRPCgcpConfig.ChannelPool.BindPickStrategy = grpcgcppb.ChannelPoolConfig_ROUND_ROBIN
 
 	cfg.DialFunc = func(ctx context.Context, target string, dopts ...grpc.DialOption) (*grpc.ClientConn, error) {
-		copts := opts
+		copts := make([]option.ClientOption, len(options))
+		copy(copts, options)
 
 		for _, do := range dopts {
 			copts = append(copts, option.WithGRPCDialOption(do))

spanner/client_test.go

@@ -281,9 +281,6 @@ func TestReadOnlyTransactionClose(t *testing.T) {
 }
 
 func TestClient_MultiEndpoint(t *testing.T) {
-	if !useGRPCgcp {
-		t.Skip("gRPC-GCP only test")
-	}
 	t.Parallel()
 
 	server, opts, serverTeardown := NewMockedSpannerInMemTestServerWithAddr(t, "localhost:0")
@@ -475,6 +472,38 @@ func TestClient_MultiEndpoint(t *testing.T) {
 	}
 }
 
+func TestClient_MultiEndpointConcurrent(t *testing.T) {
+	t.Parallel()
+
+	server, opts, serverTeardown := NewMockedSpannerInMemTestServerWithAddr(t, "localhost:0")
+	defer serverTeardown()
+
+	gmeCfg := &grpcgcp.GCPMultiEndpointOptions{
+		MultiEndpoints: map[string]*multiendpoint.MultiEndpointOptions{
+			"default": {
+				Endpoints: []string{
+					server.ServerAddress,
+				},
+			},
+		},
+		Default: "default",
+	}
+
+	ctx := context.Background()
+	formattedDatabase := fmt.Sprintf("projects/%s/instances/%s/databases/%s", "[PROJECT]", "[INSTANCE]", "[DATABASE]")
+
+	// Create multiple clients concurrently.
+	// Should not raise data race when tested with '-race'.
+	for i := 0; i < 50; i++ {
+		go func() {
+			_, _, err := NewMultiEndpointClient(ctx, formattedDatabase, gmeCfg, opts...)
+			if err != nil {
+				t.Error(err)
+			}
+		}()
+	}
+}
+
 func TestClient_MultiplexedSession(t *testing.T) {
 	var tests = []struct {
 		name     string