server: enable testserver to take different store configurations

testServerArgs will now take StoreSpecs instead of just a number of stores.
This also adds ServerArgsPerNode on testClusterArgs to enable per server
customizable settings. Currently, only in-memory stores are supported.

Part of work towards cockroachdb#8473.

base/store_spec.go

@@ -34,9 +34,9 @@ import (
 // -- 'Stores' and 'JoinList', which satisfies pflag's value interface
 
 // MinimumStoreSize is the smallest size in bytes that a store can have. This
-// number was originally based on config's defaultZoneConfig's RangeMaxBytes,
-// which is extremely stable and to avoid adding the dependency on config here,
-// it is just hard coded to 640MiB.
+// number is based on config's defaultZoneConfig's RangeMaxBytes, which is
+// extremely stable. To avoid adding the dependency on config here, it is just
+// hard coded to 640MiB.
 const MinimumStoreSize = 10 * 64 << 20
 
 // StoreSpec contains the details that can be specified in the cli pertaining

base/test_server_args.go

@@ -41,7 +41,11 @@ type TestServerArgs struct {
 	// JoinAddr (if nonempty) is the address of a node we are joining.
 	JoinAddr string
 
-	StoresPerNode int
+	// StoreSpecs define the stores for this server. If you want more than one
+	// store per node, populate this array with StoreSpecs each representing a
+	// store. If no StoreSpecs are provided than a single DefaultTestStoreSpec
+	// will be used.
+	StoreSpecs []StoreSpec
 
 	// Fields copied to the server.Context.
 	Insecure              bool
@@ -74,6 +78,19 @@ type TestClusterArgs struct {
 	ServerArgs TestServerArgs
 	// ReplicationMode controls how replication is to be done in the cluster.
 	ReplicationMode TestClusterReplicationMode
+
+	// ServerArgsPerNode override the default ServerArgs with the value in this
+	// map. The map's key is an index within TestCluster.Servers. If there is
+	// no entry in the map for a particular server, the default ServerArgs are
+	// used.
+	ServerArgsPerNode map[int]TestServerArgs
+}
+
+// DefaultTestStoreSpec is just a single in memory store of 100 MiB with no
+// special attributes.
+var DefaultTestStoreSpec = StoreSpec{
+	SizeInBytes: 100 << 20,
+	InMemory:    true,
 }
 
 // TestClusterReplicationMode represents the replication settings for a TestCluster.

kv/dist_sender_server_test.go

@@ -1162,7 +1162,12 @@ func TestPropagateTxnOnPushError(t *testing.T) {
 // should be willing to get rid of it.
 func TestRequestToUninitializedRange(t *testing.T) {
 	defer leaktest.AfterTest(t)()
-	srv, _, _ := serverutils.StartServer(t, base.TestServerArgs{StoresPerNode: 2})
+	srv, _, _ := serverutils.StartServer(t, base.TestServerArgs{
+		StoreSpecs: []base.StoreSpec{
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+		},
+	})
 	defer srv.Stopper().Stop()
 	s := srv.(*server.TestServer)
 

server/status_test.go

@@ -151,7 +151,13 @@ func TestStatusGossipJson(t *testing.T) {
 // the scan to complete, and return the server. The caller is
 // responsible for stopping the server.
 func startServer(t *testing.T) *TestServer {
-	tsI, _, kvDB := serverutils.StartServer(t, base.TestServerArgs{StoresPerNode: 3})
+	tsI, _, kvDB := serverutils.StartServer(t, base.TestServerArgs{
+		StoreSpecs: []base.StoreSpec{
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+		},
+	})
 
 	ts := tsI.(*TestServer)
 

server/testserver.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 
 	"github.com/cockroachdb/cockroach/base"
@@ -41,7 +42,6 @@ import (
 	"github.com/cockroachdb/cockroach/util/hlc"
 	"github.com/cockroachdb/cockroach/util/metric"
 	"github.com/cockroachdb/cockroach/util/stop"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -218,11 +218,24 @@ func (ts *TestServer) Start(params base.TestServerArgs) error {
 
 	// Ensure we have the correct number of engines. Add in-memory ones where
 	// needed. There must be at least one store/engine.
-	if params.StoresPerNode < 1 {
-		params.StoresPerNode = 1
+	if len(params.StoreSpecs) == 0 {
+		params.StoreSpecs = []base.StoreSpec{base.DefaultTestStoreSpec}
 	}
-	for i := len(ts.Ctx.Engines); i < params.StoresPerNode; i++ {
-		ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, params.Stopper))
+	for _, storeSpec := range params.StoreSpecs {
+		if storeSpec.InMemory {
+			if storeSpec.SizePercent > 0 {
+				panic(fmt.Sprintf("test server does not yet support in memory stores based on percentage of total memory: %s", storeSpec))
+			}
+			ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(
+				roachpb.Attributes{},
+				storeSpec.SizeInBytes,
+				params.Stopper,
+			))
+		} else {
+			// TODO(bram): This will require both cleanup and on failure or if
+			// desired, not cleanup (perhaps on failure).
+			panic(fmt.Sprintf("test server does not yet support in non-memory stores: %s", storeSpec))
+		}
 	}
 
 	var err error

server/updates_test.go

@@ -95,7 +95,12 @@ func TestReportUsage(t *testing.T) {
 	}
 	defer stubURL(&reportingURL, u)()
 
-	params := base.TestServerArgs{StoresPerNode: 2}
+	params := base.TestServerArgs{
+		StoreSpecs: []base.StoreSpec{
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+		},
+	}
 	s, _, _ := serverutils.StartServer(t, params)
 	ts := s.(*TestServer)
 
@@ -144,7 +149,7 @@ func TestReportUsage(t *testing.T) {
 	if minExpected, actual := totalRanges, reported.Node.RangeCount; minExpected > actual {
 		t.Errorf("expected node ranges at least %v got %v", minExpected, actual)
 	}
-	if minExpected, actual := params.StoresPerNode, len(reported.Stores); minExpected > actual {
+	if minExpected, actual := len(params.StoreSpecs), len(reported.Stores); minExpected > actual {
 		t.Errorf("expected at least %v stores got %v", minExpected, actual)
 	}
 

testutils/testcluster/testcluster.go

@@ -136,7 +136,13 @@ func StartTestCluster(t testing.TB, nodes int, args base.TestClusterArgs) *TestC
 
 	args.ServerArgs.PartOfCluster = true
 	for i := 0; i < nodes; i++ {
-		serverArgs := args.ServerArgs
+		var serverArgs base.TestServerArgs
+		if perNodeServerArgs, ok := args.ServerArgsPerNode[i]; ok {
+			serverArgs = perNodeServerArgs
+			serverArgs.PartOfCluster = true
+		} else {
+			serverArgs = args.ServerArgs
+		}
 		serverArgs.Stopper = stop.NewStopper()
 		if i > 0 {
 			serverArgs.JoinAddr = tc.Servers[0].ServingAddr()