From 7340f6248d7035e17c109e39e9e31a8bfc9f532c Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Wed, 26 Apr 2023 20:02:49 +0000 Subject: [PATCH 1/6] split: convert example tests into datadriven There were existing example tests for evaluating the load based splitter. These were not a part of testing but instead could be run ad-hoc for experiments. This commit updates these tests to be datadriven in order to spot regressions easily in the load based splitter. The datadriven interface may also be used for experimental evaluation in an ad-hoc manner, similar to before. Release note: None --- pkg/kv/kvserver/split/BUILD.bazel | 4 + .../split/load_based_splitter_test.go | 914 ++++++++++++++---- .../split/testdata/cpu_decider_cartesian | 131 +++ .../kvserver/split/testdata/unweighted_finder | 10 + .../kvserver/split/testdata/weighted_finder | 49 + 5 files changed, 906 insertions(+), 202 deletions(-) create mode 100644 pkg/kv/kvserver/split/testdata/cpu_decider_cartesian create mode 100644 pkg/kv/kvserver/split/testdata/unweighted_finder create mode 100644 pkg/kv/kvserver/split/testdata/weighted_finder diff --git a/pkg/kv/kvserver/split/BUILD.bazel b/pkg/kv/kvserver/split/BUILD.bazel index 26de0dd675e3..8ceee186faa5 100644 --- a/pkg/kv/kvserver/split/BUILD.bazel +++ b/pkg/kv/kvserver/split/BUILD.bazel @@ -31,16 +31,20 @@ go_test( "weighted_finder_test.go", ], args = ["-test.timeout=55s"], + data = glob(["testdata/**"]), embed = [":split"], deps = [ "//pkg/keys", "//pkg/roachpb", + "//pkg/testutils/datapathutils", + "//pkg/testutils/skip", "//pkg/util/encoding", "//pkg/util/leaktest", "//pkg/util/metric", "//pkg/util/stop", "//pkg/util/timeutil", "//pkg/workload/ycsb", + "@com_github_cockroachdb_datadriven//:datadriven", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@org_golang_x_exp//rand", diff --git a/pkg/kv/kvserver/split/load_based_splitter_test.go b/pkg/kv/kvserver/split/load_based_splitter_test.go index e043715a9735..07eeecceaa85 100644 --- a/pkg/kv/kvserver/split/load_based_splitter_test.go +++ b/pkg/kv/kvserver/split/load_based_splitter_test.go @@ -12,16 +12,23 @@ package split import ( "bytes" + "context" "fmt" "math" "sort" + "testing" "text/tabwriter" "time" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/testutils/datapathutils" + "github.com/cockroachdb/cockroach/pkg/testutils/skip" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/metric" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/workload/ycsb" + "github.com/cockroachdb/datadriven" "golang.org/x/exp/rand" ) @@ -73,24 +80,23 @@ import ( // the split key quality of the load splitter. const ( - zipfGenerator = 0 - uniformGenerator = 1 - numIterations = 200 + zipfGenerator = 0 + uniformGenerator = 1 + defaultIterations = 200 ) -type generator interface { - Uint64() uint64 +func genToString(gen int) string { + switch gen { + case zipfGenerator: + return "zip" + case uniformGenerator: + return "uni" + default: + panic("unknown gen") + } } -type config struct { - lbs LoadBasedSplitter - startKeyGenerator generator - spanLengthGenerator generator - weightGenerator generator - rangeRequestPercent float64 - numRequests int - randSource *rand.Rand -} +var startTime = time.Date(2022, 03, 21, 11, 0, 0, 0, time.UTC) type request struct { span roachpb.Span @@ -102,8 +108,46 @@ type weightedKey struct { weight float64 } -type lbsTestSettings struct { - desc string +type generator interface { + Uint64() uint64 +} + +func newGenerator(randSource *rand.Rand, generatorType int, iMax uint64) generator { + var g generator + var err error + if generatorType == zipfGenerator { + g, err = ycsb.NewZipfGenerator(randSource, 1, iMax, 0.99, false) + } else if generatorType == uniformGenerator { + g, err = ycsb.NewUniformGenerator(randSource, 1, iMax) + } else { + panic("generatorType must be zipfGenerator or uniformGenerator") + } + if err != nil { + panic(err) + } + return g +} + +// We want to apply a weight distribution over the keyspace, where the +// weight(key)/sum(weight(keys)) is some fraction representing the relative +// load on the key. We also want to apply an access distribution over the +// keyspace, where keys are accessed with some probability, semi-independently +// of their weights. If a key is never accessed it cannot have weight, so the +// weight and access are not independent. For example, the weight and access +// distributions may look something like this: +// +// weight distribution access distribution +// +// |x | +// |x x | +// w |x x x x w | +// |x x x x x x |x x x x x x +// +----------- +----------- +// k k +// +// In order to get these two distributions, multiple request configs are +// necessary. Which can then be mixed with mixGenerators. +type requestConfig struct { startKeyGeneratorType int startKeyGeneratorIMax uint64 spanLengthGeneratorType int @@ -112,27 +156,86 @@ type lbsTestSettings struct { weightGeneratorIMax uint64 rangeRequestPercent float64 numRequests int - lbs func(*rand.Rand) LoadBasedSplitter - seed uint64 } -func uint32ToKey(key uint32) roachpb.Key { - return keys.SystemSQLCodec.TablePrefix(key) +func (rc requestConfig) String() string { + return fmt.Sprintf("w=%s(%d)/k=%s(%d)/s=%s(%d)/s(%%)=%d/%d", + genToString(rc.weightGeneratorType), rc.weightGeneratorIMax, + genToString(rc.startKeyGeneratorType), rc.startKeyGeneratorIMax, + genToString(rc.spanLengthGeneratorType), rc.spanLengthGeneratorIMax, + int(rc.rangeRequestPercent*100), + rc.numRequests, + ) +} + +func (rc requestConfig) makeGenerator(randSource *rand.Rand) requestGenerator { + return requestGenerator{ + startKeyGenerator: newGenerator(randSource, rc.startKeyGeneratorType, rc.startKeyGeneratorIMax), + spanLengthGenerator: newGenerator(randSource, rc.spanLengthGeneratorType, rc.spanLengthGeneratorIMax), + weightGenerator: newGenerator(randSource, rc.weightGeneratorType, rc.weightGeneratorIMax), + rangeRequestPercent: rc.rangeRequestPercent, + numRequests: rc.numRequests, + } +} + +type multiReqConfig struct { + reqConfigs []requestConfig + mix mixType +} + +func (mrc multiReqConfig) makeGenerator(randSource *rand.Rand) multiRequestGenerator { + gens := make([]requestGenerator, 0, len(mrc.reqConfigs)) + var numRequests int + for _, reqConfig := range mrc.reqConfigs { + gen := reqConfig.makeGenerator(randSource) + numRequests += gen.numRequests + gens = append(gens, gen) + } + + return multiRequestGenerator{ + requestGenerators: gens, + mix: mrc.mix, + rand: randSource, + numRequests: numRequests, + } +} + +type mixType int + +const ( + sequential mixType = iota + permute +) + +func (mrc multiReqConfig) String() string { + var buf bytes.Buffer + for _, rc := range mrc.reqConfigs { + fmt.Fprintf(&buf, "%s\n", rc) + } + return buf.String() } -func generateRequests(config *config) ([]request, []weightedKey, float64) { +type requestGenerator struct { + startKeyGenerator generator + spanLengthGenerator generator + weightGenerator generator + rangeRequestPercent float64 + numRequests int +} + +func (rg requestGenerator) generate() ([]request, []weightedKey, float64) { var totalWeight float64 - requests := make([]request, 0, config.numRequests) - weightedKeys := make([]weightedKey, 0, 2*config.numRequests) + requests := make([]request, 0, rg.numRequests) + weightedKeys := make([]weightedKey, 0, 2*rg.numRequests) - for i := 0; i < config.numRequests; i++ { - startKey := uint32(config.startKeyGenerator.Uint64()) - spanLength := uint32(config.spanLengthGenerator.Uint64()) - weight := float64(config.weightGenerator.Uint64()) + for i := 0; i < rg.numRequests; i++ { + startKey := uint32(rg.startKeyGenerator.Uint64()) + spanLength := uint32(rg.spanLengthGenerator.Uint64()) + weight := float64(rg.weightGenerator.Uint64()) var span roachpb.Span span.Key = uint32ToKey(startKey) - if config.randSource.Float64() < config.rangeRequestPercent { + if rand.Float64() < rg.rangeRequestPercent { endKey := startKey + spanLength span.EndKey = uint32ToKey(endKey) @@ -159,6 +262,92 @@ func generateRequests(config *config) ([]request, []weightedKey, float64) { return requests, weightedKeys, totalWeight } +type multiRequestGenerator struct { + requestGenerators []requestGenerator + mix mixType + rand *rand.Rand + numRequests int +} + +func (mrg multiRequestGenerator) generate() ([]request, []weightedKey, float64) { + numRequests := 0 + for _, gen := range mrg.requestGenerators { + numRequests += gen.numRequests + } + + var totalWeight float64 + requests := make([]request, 0, numRequests) + weightedKeys := make([]weightedKey, 0, numRequests*2) + for _, gen := range mrg.requestGenerators { + reqs, keys, weight := gen.generate() + requests = append(requests, reqs...) + weightedKeys = append(weightedKeys, keys...) + totalWeight += weight + } + + switch mrg.mix { + case sequential: + // Nothing to do. + case permute: + perms := mrg.rand.Perm(numRequests) + permutedRequests := make([]request, numRequests) + for i, idx := range perms { + permutedRequests[idx] = requests[i] + } + requests = permutedRequests + default: + panic("unknown mix type") + } + + return requests, weightedKeys, totalWeight +} + +type finderConfig struct { + weighted bool +} + +func (fc finderConfig) makeFinder(randSource rand.Source) LoadBasedSplitter { + if fc.weighted { + return NewWeightedFinder(startTime, rand.New(randSource)) + } + return NewUnweightedFinder(startTime, rand.New(randSource)) +} + +type deciderConfig struct { + threshold float64 + retention time.Duration + objective SplitObjective + duration time.Duration +} + +func (dc deciderConfig) makeDecider(randSource rand.Source) *Decider { + d := &Decider{} + loadSplitConfig := testLoadSplitConfig{ + randSource: rand.New(randSource), + useWeighted: dc.objective == SplitCPU, + statRetention: dc.retention, + statThreshold: dc.threshold, + } + + Init(d, &loadSplitConfig, &LoadSplitterMetrics{ + PopularKeyCount: metric.NewCounter(metric.Metadata{}), + NoSplitKeyCount: metric.NewCounter(metric.Metadata{}), + }, dc.objective) + return d +} + +type lbsTestSettings struct { + requestConfig multiReqConfig + deciderConfig *deciderConfig + finderConfig *finderConfig + seed uint64 + iterations int +} + +func uint32ToKey(key uint32) roachpb.Key { + return keys.SystemSQLCodec.TablePrefix(key) +} + func getOptimalKey( weightedKeys []weightedKey, totalWeight float64, ) (optimalKey uint32, optimalLeftWeight, optimalRightWeight float64) { @@ -185,22 +374,51 @@ func getOptimalKey( } func getKey( - config *config, requests []request, weightedKeys []weightedKey, + recordFn func(span roachpb.Span, weight int), keyFn func() roachpb.Key, requests []request, ) ( key uint32, leftWeight, rightWeight float64, recordExecutionTime, keyExecutionTime time.Duration, + noKeyFound bool, ) { recordStart := timeutil.Now() for _, request := range requests { - config.lbs.Record(request.span, request.weight) + recordFn(request.span, int(request.weight)) } recordExecutionTime = timeutil.Since(recordStart) keyStart := timeutil.Now() - keyRoachpbKey := config.lbs.Key() + keyRoachpbKey := keyFn() keyExecutionTime = timeutil.Since(keyStart) + noKeyFound = keyRoachpbKey.Equal(keys.MinKey) _, key, _ = keys.SystemSQLCodec.DecodeTablePrefix(keyRoachpbKey) + return +} + +type result struct { + key, optimalKey uint32 + leftWeight, rightWeight, optimalLeftWeight, optimalRightWeight float64 + recordExecutionTime, keyExecutionTime time.Duration + noKeyFound bool +} + +func runTest( + recordFn func(span roachpb.Span, weight int), + keyFn func() roachpb.Key, + randSource rand.Source, + mrg multiRequestGenerator, +) result { + requests, weightedKeys, totalWeight := mrg.generate() + optimalKey, optimalLeftWeight, optimalRightWeight := getOptimalKey( + weightedKeys, + totalWeight, + ) + key, leftWeight, rightWeight, recordExecutionTime, + keyExecutionTime, noKeyFound := getKey( + recordFn, + keyFn, + requests, + ) for _, weightedKey := range weightedKeys { if weightedKey.key < key { leftWeight += weightedKey.weight @@ -208,195 +426,487 @@ func getKey( rightWeight += weightedKey.weight } } - return + return result{ + key: key, + optimalKey: optimalKey, + leftWeight: leftWeight, + rightWeight: rightWeight, + optimalLeftWeight: optimalLeftWeight, + optimalRightWeight: optimalRightWeight, + recordExecutionTime: recordExecutionTime, + keyExecutionTime: keyExecutionTime, + noKeyFound: noKeyFound, + } } -func runTest( - config *config, -) ( - key, optimalKey uint32, - leftWeight, rightWeight, optimalLeftWeight, optimalRightWeight float64, - recordExecutionTime, keyExecutionTime time.Duration, -) { - requests, weightedKeys, totalWeight := generateRequests(config) - optimalKey, optimalLeftWeight, optimalRightWeight = getOptimalKey(weightedKeys, totalWeight) - key, leftWeight, rightWeight, recordExecutionTime, keyExecutionTime = getKey(config, requests, weightedKeys) - return +type repeatedResult struct { + avgPercentDifference, maxPercentDifference, + avgOptimalPercentDifference, maxOptimalPercentDifference float64 + avgRecordExecutionTime, avgKeyExecutionTime time.Duration + noKeyFoundPercent float64 } -func newGenerator(randSource *rand.Rand, generatorType int, iMax uint64) generator { - var g generator - var err error - if generatorType == zipfGenerator { - g, err = ycsb.NewZipfGenerator(randSource, 1, iMax, 0.99, false) - } else if generatorType == uniformGenerator { - g, err = ycsb.NewUniformGenerator(randSource, 1, iMax) - } else { - panic("generatorType must be zipfGenerator or uniformGenerator") +func resultTable(configs []multiReqConfig, rr []repeatedResult, showTiming bool) string { + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 4, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(w, "description\t"+ + "no_key(%)\t"+ + "avg_diff(%)\t"+ + "max_diff(%)\t"+ + "avg_optimal_diff(%)\t"+ + "max_optimal_diff(%)", + ) + if showTiming { + _, _ = fmt.Fprintln(w, + "\tavg_record_time\t"+ + "avg_key_time", + ) } - if err != nil { - panic(err) + + for i, r := range rr { + n := len(configs[i].reqConfigs) + var desc string + // When there are multiple descriptions, print out the results in the + // first row and the rest of the descriptions in subsequent rows with + // empty results. + if n > 1 { + desc = fmt.Sprintf("mixed_requests(%d)", n) + } else { + desc = configs[i].reqConfigs[0].String() + } + + _, _ = fmt.Fprintf(w, "%s\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f", + desc, + r.noKeyFoundPercent, + r.avgPercentDifference, + r.maxPercentDifference, + r.avgOptimalPercentDifference, + r.maxOptimalPercentDifference, + ) + + if showTiming { + _, _ = fmt.Fprintf(w, "\t%s\t%s", + r.avgRecordExecutionTime, + r.avgKeyExecutionTime, + ) + } + + _, _ = fmt.Fprintln(w) + + // We already printed the results and description for the single config + // case. + if n == 1 { + continue + } + // Print the descriptions for the multi-config case. + for _, reqConfig := range configs[i].reqConfigs { + _, _ = fmt.Fprintf(w, "%s\t\t\t\t\t\n", reqConfig) + } } - return g + _ = w.Flush() + + return buf.String() + } -func runTestRepeated( - settings *lbsTestSettings, -) ( - avgPercentDifference, maxPercentDifference, avgOptimalPercentDifference, maxOptimalPercentDifference float64, - avgRecordExecutionTime, avgKeyExecutionTime time.Duration, -) { +func runTestRepeated(settings *lbsTestSettings) repeatedResult { + var ( + avgPercentDifference, maxPercentDifference, + avgOptimalPercentDifference, maxOptimalPercentDifference float64 + avgRecordExecutionTime, avgKeyExecutionTime time.Duration + noKeyFoundPercent float64 + ) randSource := rand.New(rand.NewSource(settings.seed)) - startKeyGenerator := newGenerator(randSource, settings.startKeyGeneratorType, settings.startKeyGeneratorIMax) - spanLengthGenerator := newGenerator(randSource, settings.spanLengthGeneratorType, settings.spanLengthGeneratorIMax) - weightGenerator := newGenerator(randSource, settings.weightGeneratorType, settings.weightGeneratorIMax) - for i := 0; i < numIterations; i++ { - _, _, leftWeight, rightWeight, optimalLeftWeight, optimalRightWeight, recordExecutionTime, keyExecutionTime := runTest(&config{ - lbs: settings.lbs(randSource), - startKeyGenerator: startKeyGenerator, - spanLengthGenerator: spanLengthGenerator, - weightGenerator: weightGenerator, - rangeRequestPercent: settings.rangeRequestPercent, - numRequests: settings.numRequests, - randSource: randSource, - }) - percentDifference := 100 * math.Abs(leftWeight-rightWeight) / (leftWeight + rightWeight) - avgPercentDifference += percentDifference - if maxPercentDifference < percentDifference { - maxPercentDifference = percentDifference + requestGen := settings.requestConfig.makeGenerator(randSource) + + for i := 0; i < settings.iterations; i++ { + var recordFn func(span roachpb.Span, weight int) + var keyFn func() roachpb.Key + + if settings.deciderConfig != nil { + d := settings.deciderConfig.makeDecider(randSource) + now := startTime + duration := settings.deciderConfig.duration + requestInterval := duration / time.Duration(requestGen.numRequests) + + ctx := context.Background() + recordFn = func(span roachpb.Span, weight int) { + d.Record(ctx, now, + func(SplitObjective) int { return weight }, + func() roachpb.Span { return span }, + ) + now = now.Add(requestInterval) + } + keyFn = func() roachpb.Key { + return d.MaybeSplitKey(ctx, now) + } + } else { + f := settings.finderConfig.makeFinder(randSource) + recordFn = func(span roachpb.Span, weight int) { + f.Record(span, float64(weight)) + } + keyFn = func() roachpb.Key { + return f.Key() + } } - optimalPercentDifference := 100 * math.Abs(optimalLeftWeight-optimalRightWeight) / (optimalLeftWeight + optimalRightWeight) - avgOptimalPercentDifference += optimalPercentDifference - if maxOptimalPercentDifference < optimalPercentDifference { - maxOptimalPercentDifference = optimalPercentDifference + ret := runTest(recordFn, keyFn, randSource, requestGen) + + if !ret.noKeyFound { + percentDifference := 100 * math.Abs(ret.leftWeight-ret.rightWeight) / (ret.leftWeight + ret.rightWeight) + avgPercentDifference += percentDifference + if maxPercentDifference < percentDifference { + maxPercentDifference = percentDifference + } + optimalPercentDifference := 100 * + math.Abs(ret.optimalLeftWeight-ret.optimalRightWeight) / + (ret.optimalLeftWeight + ret.optimalRightWeight) + + avgOptimalPercentDifference += optimalPercentDifference + if maxOptimalPercentDifference < optimalPercentDifference { + maxOptimalPercentDifference = optimalPercentDifference + } + } else { + noKeyFoundPercent++ } - avgRecordExecutionTime += recordExecutionTime - avgKeyExecutionTime += keyExecutionTime + avgRecordExecutionTime += ret.recordExecutionTime + avgKeyExecutionTime += ret.keyExecutionTime + + } + + nRuns := settings.iterations + keyRuns := nRuns - int(noKeyFoundPercent) + + avgRecordExecutionTime = time.Duration(avgRecordExecutionTime.Nanoseconds() / int64(nRuns)) + avgKeyExecutionTime = time.Duration(avgKeyExecutionTime.Nanoseconds() / int64(nRuns)) + avgPercentDifference /= float64(keyRuns) + avgOptimalPercentDifference /= float64(keyRuns) + noKeyFoundPercent /= float64(nRuns) + // We want a percent (0-100). + noKeyFoundPercent *= 100 + + return repeatedResult{ + avgPercentDifference: avgPercentDifference, + maxPercentDifference: maxPercentDifference, + avgOptimalPercentDifference: avgOptimalPercentDifference, + maxOptimalPercentDifference: maxOptimalPercentDifference, + avgRecordExecutionTime: avgRecordExecutionTime, + avgKeyExecutionTime: avgKeyExecutionTime, + noKeyFoundPercent: noKeyFoundPercent, } - avgRecordExecutionTime = time.Duration(avgRecordExecutionTime.Nanoseconds() / int64(numIterations)) - avgKeyExecutionTime = time.Duration(avgKeyExecutionTime.Nanoseconds() / int64(numIterations)) - avgPercentDifference /= numIterations - avgOptimalPercentDifference /= numIterations - return } -func runTestMultipleSettings(settingsArr []lbsTestSettings) { - var buf bytes.Buffer - w := tabwriter.NewWriter(&buf, 4, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(w, - "Description\t"+ - "Avg Percent Difference\t"+ - "Max Percent Difference\t"+ - "Avg Optimal Percent Difference\t"+ - "Max Optimal Percent Difference\t"+ - "Avg Record Execution Time\t"+ - "Avg Key Execution Time", - ) - for _, settings := range settingsArr { - avgPercentDifference, maxPercentDifference, avgOptimalPercentDifference, maxOptimalPercentDifference, avgRecordExecutionTime, avgKeyExecutionTime := runTestRepeated(&settings) - _, _ = fmt.Fprintf(w, "%s\t%f\t%f\t%f\t%f\t%s\t%s\n", - settings.desc, - avgPercentDifference, - maxPercentDifference, - avgOptimalPercentDifference, - maxOptimalPercentDifference, - avgRecordExecutionTime, - avgKeyExecutionTime) +func cartesianProduct(lists [][]int) [][]int { + // Base case: if there's only one list, return it as a nested list. + if len(lists) == 1 { + result := [][]int{} + for _, elem := range lists[0] { + result = append(result, []int{elem}) + } + return result } - _ = w.Flush() - fmt.Print(buf.String()) -} - -func ExampleUnweightedFinder() { - runTestMultipleSettings([]lbsTestSettings{ - { - desc: "UnweightedFinder", - startKeyGeneratorType: zipfGenerator, - startKeyGeneratorIMax: 10000000000, - spanLengthGeneratorType: uniformGenerator, - spanLengthGeneratorIMax: 1000, - weightGeneratorType: uniformGenerator, - weightGeneratorIMax: 1, - rangeRequestPercent: 0.95, - numRequests: 13000, - lbs: func(randSource *rand.Rand) LoadBasedSplitter { - return NewUnweightedFinder(timeutil.Now(), randSource) - }, - seed: 2022, - }, + + // Recursive case: compute the cartesian product of the first list and the + // product of the rest of the lists + rest := cartesianProduct(lists[1:]) + result := [][]int{} + for _, elem := range lists[0] { + for _, partial := range rest { + result = append(result, append([]int{elem}, partial...)) + } + } + return result +} + +func enumerateBasicRequestConfigs() []requestConfig { + distributions := []int{uniformGenerator, zipfGenerator} + spanLengths := []int{1, 1000} + startKeyMaxes := []int{10000, 1000000} + weightMaxes := []int{100, 10000} + rangePercents := []int{0, 20, 95} + + cartesian := cartesianProduct([][]int{ + distributions, // Weight distribution. + weightMaxes, // Weight max. + distributions, // Start key distribution. + startKeyMaxes, // Start key max. + distributions, // Span distribution. + spanLengths, // Span length max. + rangePercents, // Range request percent. }) + + configMap := map[requestConfig]struct{}{} + + configs := make([]requestConfig, 0, len(cartesian)/2) + for _, c := range cartesian { + config := requestConfig{ + weightGeneratorType: c[0], + weightGeneratorIMax: uint64(c[1]), + startKeyGeneratorType: c[2], + startKeyGeneratorIMax: uint64(c[3]), + spanLengthGeneratorType: c[4], + spanLengthGeneratorIMax: uint64(c[5]), + rangeRequestPercent: float64(c[6]) / 100.0, + // Always use 10k requests for simplicity. + numRequests: 10000, + } + + // Don't include configurations where there's a non-zero span percent but + // no span length or vice versa. + if config.spanLengthGeneratorIMax > 1 && config.rangeRequestPercent == 0 || + (config.spanLengthGeneratorIMax <= 1 && config.rangeRequestPercent > 0) { + continue + } + // No point running both zipfian and uniform for unweighted, only use + // uniform. + if config.weightGeneratorIMax == 1 && config.weightGeneratorType == zipfGenerator { + continue + } + + // No point running both zipfian and uniform spans for 1 span length. + if config.spanLengthGeneratorIMax == 1 && + config.spanLengthGeneratorType == zipfGenerator { + continue + } + + // Don't include duplicate entries. + if _, present := configMap[config]; present { + continue + } + configMap[config] = struct{}{} + configs = append(configs, config) + } + return configs +} + +func makeMultiRequestConfigs( + seed uint64, mixCount int, mix mixType, requestConfigs ...requestConfig, +) []multiReqConfig { + ret := []multiReqConfig{} + if mixCount == 0 { + for _, reqConfig := range requestConfigs { + ret = append(ret, multiReqConfig{ + reqConfigs: []requestConfig{reqConfig}, + mix: mix, + }) + } + } else { + // This is expensive, we could instead pass in the source. + n := len(requestConfigs) + rand := rand.New(rand.NewSource(seed)) + perms := rand.Perm(n) + for i := 0; i < n; i += mixCount { + mrc := multiReqConfig{ + mix: mix, + reqConfigs: make([]requestConfig, 0, mixCount), + } + for j := 0; j < mixCount && i+j < n; j++ { + mrc.reqConfigs = append(mrc.reqConfigs, requestConfigs[perms[i+j]]) + } + ret = append(ret, mrc) + } + } + return ret } -func ExampleWeightedFinder() { - seed := uint64(2022) - lbs := func(randSource *rand.Rand) LoadBasedSplitter { - return NewWeightedFinder(timeutil.Now(), randSource) +// TestDataDriven is a data-driven test for testing the integrated load +// splitter. It allows testing the results of different request distributions +// and sampling durations against the optimal split key. The commands provided +// are: +// +// - "requests" key_dist=(zipfian|uniform) key_max= +// span_dist=(zipfian|uniform) span_max= weight_dist=(zipfian|uniform) +// weight_max= range_request_percent= request_count= +// key_dist is the distribution of start keys, key_max is the maximum start +// key, span_dist is the distribution of span lengths (offset by the start +// key), span_max is the maximum span length, weight_dist is the distribution +// of span weights, weight_max is the maximum span weight, +// range_request_percent is the percentage of requests which will be spanning +// i.e. non-point requests and request_count is the number of requests to +// generate. +// +// - "decider" duration= threshold= retention= +// objective=(cpu|qps) duration is the time in seconds the decider will +// receive requests over, threshold is number above which the decider will +// instantiate a key finder, retention is the time that requests are tracked +// for in the sliding max window and objective is the split objective - which +// acts the same as weighted vs unweighted for CPU and QPS. +// +// The decider is the container and manager of the finder, if this is +// specified, then a finder will be instantiated on the next call to eval. It +// is added here for completeness and e2e testing of the pkg. +// +// - "finder" weighted= weighted indicates the finder should be the +// weighted variation. +// +// The finder, resolves finding an actual split key given incoming requests. +// The finder does not have notions of timing, unlike the decider. +// +// - "eval" seed= iterations= cartesian= +// [mix=(sequential|perm) mix_count=] seed is the seed to instantiate +// the current finder/decider and request generators with, iterations is the +// number of iterations to repeatedly run before synthesizing a result, +// cartesian evaluates a list of configurations with every different +// distribution and a few maximum values for span length, start key and +// weight, mix and mix_count mix loaded requests together and evaluate them +// in one test; mixing can be used to generate different access and weight +// distributions. +// +// To test multiple concurrent splitters in a cluster, see +// asim/tests/testdata/example_splitting, which enables larger testing. +func TestDataDriven(t *testing.T) { + defer leaktest.AfterTest(t)() + skip.UnderShort(t, "takes 20s") + skip.UnderStressRace(t, "takes 20s") + + parseGeneratorType := func(dist string) int { + switch dist { + case "zipfian": + return zipfGenerator + case "uniform": + return uniformGenerator + default: + panic("unknown gen type") + } + } + + parseObjType := func(obj string) SplitObjective { + switch obj { + case SplitCPU.String(): + return SplitCPU + case SplitQPS.String(): + return SplitQPS + default: + panic("unknown split objective") + } } - runTestMultipleSettings([]lbsTestSettings{ - { - desc: "WeightedFinder/startIMax=10000000000/spanIMax=1000", - startKeyGeneratorType: zipfGenerator, - startKeyGeneratorIMax: 10000000000, - spanLengthGeneratorType: uniformGenerator, - spanLengthGeneratorIMax: 1000, - weightGeneratorType: uniformGenerator, - weightGeneratorIMax: 10, - rangeRequestPercent: 0.95, - numRequests: 10000, - lbs: lbs, - seed: seed, - }, - { - desc: "WeightedFinder/startIMax=100000/spanIMax=1000", - startKeyGeneratorType: zipfGenerator, - startKeyGeneratorIMax: 100000, - spanLengthGeneratorType: uniformGenerator, - spanLengthGeneratorIMax: 1000, - weightGeneratorType: uniformGenerator, - weightGeneratorIMax: 10, - rangeRequestPercent: 0.95, - numRequests: 10000, - lbs: lbs, - seed: seed, - }, - { - desc: "WeightedFinder/startIMax=1000/spanIMax=100", - startKeyGeneratorType: zipfGenerator, - startKeyGeneratorIMax: 1000, - spanLengthGeneratorType: uniformGenerator, - spanLengthGeneratorIMax: 100, - weightGeneratorType: uniformGenerator, - weightGeneratorIMax: 10, - rangeRequestPercent: 0.95, - numRequests: 10000, - lbs: lbs, - seed: seed, - }, - { - desc: "WeightedFinder/startIMax=100000/spanIMax=1000/point", - startKeyGeneratorType: zipfGenerator, - startKeyGeneratorIMax: 100000, - spanLengthGeneratorType: uniformGenerator, - spanLengthGeneratorIMax: 1000, - weightGeneratorType: uniformGenerator, - weightGeneratorIMax: 10, - rangeRequestPercent: 0, - numRequests: 10000, - lbs: lbs, - seed: seed, - }, - { - desc: "WeightedFinder/startIMax=10000000000/spanIMax=1000/unweighted", - startKeyGeneratorType: zipfGenerator, - startKeyGeneratorIMax: 10000000000, - spanLengthGeneratorType: uniformGenerator, - spanLengthGeneratorIMax: 1000, - weightGeneratorType: uniformGenerator, - weightGeneratorIMax: 1, - rangeRequestPercent: 0.95, - numRequests: 10000, - lbs: lbs, - seed: seed, - }, + + dir := datapathutils.TestDataPath(t, "") + datadriven.Walk(t, dir, func(t *testing.T, path string) { + reqConfigs := []requestConfig{} + var decConfig *deciderConfig + var findConfig *finderConfig + datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { + switch d.Cmd { + case "requests": + var keyDist, spanDist, weightDist string + var keyIMax, spanIMax, weightIMax int + var rangeRequestPercent, requestCount int + + d.ScanArgs(t, "key_dist", &keyDist) + d.ScanArgs(t, "span_dist", &spanDist) + d.ScanArgs(t, "weight_dist", &weightDist) + + d.ScanArgs(t, "key_max", &keyIMax) + d.ScanArgs(t, "span_max", &spanIMax) + d.ScanArgs(t, "weight_max", &weightIMax) + + d.ScanArgs(t, "range_request_percent", &rangeRequestPercent) + d.ScanArgs(t, "request_count", &requestCount) + + reqConfigs = append(reqConfigs, requestConfig{ + startKeyGeneratorType: parseGeneratorType(keyDist), + startKeyGeneratorIMax: uint64(keyIMax), + spanLengthGeneratorType: parseGeneratorType(spanDist), + spanLengthGeneratorIMax: uint64(spanIMax), + weightGeneratorType: parseGeneratorType(weightDist), + weightGeneratorIMax: uint64(weightIMax), + rangeRequestPercent: float64(rangeRequestPercent) / 100.0, + numRequests: requestCount, + }) + return "" + case "finder": + var weighted bool + d.ScanArgs(t, "weighted", &weighted) + + findConfig = &finderConfig{ + weighted: weighted, + } + + case "decider": + var duration int + var threshold int + var retentionSeconds, durationSeconds int + var objective string + + d.ScanArgs(t, "duration", &duration) + d.ScanArgs(t, "retention", &retentionSeconds) + d.ScanArgs(t, "duration", &durationSeconds) + d.ScanArgs(t, "objective", &objective) + d.ScanArgs(t, "threshold", &threshold) + splitObj := parseObjType(objective) + + decConfig = &deciderConfig{ + threshold: float64(threshold), + retention: time.Duration(retentionSeconds) * time.Second, + objective: splitObj, + duration: time.Duration(duration) * time.Second, + } + case "eval": + var seed uint64 + var iterations, mixCount int + var showTiming, cartesian, all bool + var mix string + var mixT mixType + + d.ScanArgs(t, "seed", &seed) + d.ScanArgs(t, "iterations", &iterations) + if d.HasArg("timing") { + d.ScanArgs(t, "timing", &showTiming) + } + if d.HasArg("cartesian") { + d.ScanArgs(t, "cartesian", &cartesian) + } + if d.HasArg("all") { + d.ScanArgs(t, "all", &all) + } + if d.HasArg("mix") { + d.ScanArgs(t, "mix", &mix) + d.ScanArgs(t, "mix_count", &mixCount) + switch mix { + case "sequential": + mixT = sequential + case "perm": + mixT = permute + default: + panic("unknown mix type") + } + } + + n := len(reqConfigs) + var requestConfigs []requestConfig + if cartesian { + requestConfigs = enumerateBasicRequestConfigs() + } else if all { + requestConfigs = reqConfigs + } else { + if n == 0 { + panic("no request config specified") + } + if mixCount > 0 { + requestConfigs = reqConfigs[n-mixCount : n] + } else { + requestConfigs = reqConfigs[n-1 : n] + } + } + + evalRequestConfigs := makeMultiRequestConfigs( + seed, mixCount, mixT, requestConfigs...) + + repeatedResults := make([]repeatedResult, len(evalRequestConfigs)) + for i, r := range evalRequestConfigs { + repeatedResults[i] = runTestRepeated(&lbsTestSettings{ + requestConfig: r, + deciderConfig: decConfig, + finderConfig: findConfig, + seed: seed, + iterations: iterations, + }) + } + return resultTable(evalRequestConfigs, repeatedResults, showTiming) + default: + return fmt.Sprintf("unknown command: %s", d.Cmd) + } + return "" + }) }) } diff --git a/pkg/kv/kvserver/split/testdata/cpu_decider_cartesian b/pkg/kv/kvserver/split/testdata/cpu_decider_cartesian new file mode 100644 index 000000000000..1799811f27f6 --- /dev/null +++ b/pkg/kv/kvserver/split/testdata/cpu_decider_cartesian @@ -0,0 +1,131 @@ +# In this test we run many different start key, span length and weight +# distributions. We set the duration to be 100 seconds, with 20k requests ( +# mixing 2 10k request configs) there should be 200 requests/second with +# varying weights. The threshold is set low enough that most runs should exceed +# the threshold at every second. +decider duration=100 retention=200 objective=cpu threshold=1000 +---- + +eval seed=42 iterations=20 cartesian=true mix=perm mix_count=2 +---- +description no_key(%) avg_diff(%) max_diff(%) avg_optimal_diff(%) max_optimal_diff(%) +mixed_requests(2) 0.00 91.53 97.71 0.01 0.01 +w=uni(10000)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 +w=zip(100)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 18.46 25.86 0.03 0.09 +w=uni(100)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 +w=zip(10000)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 6.49 18.32 0.02 0.08 +w=zip(10000)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 +w=uni(100)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 5.56 17.30 0.00 0.01 +w=uni(10000)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 +w=uni(100)/k=uni(10000)/s=uni(1)/s(%)=0/10000 +mixed_requests(2) 0.00 90.54 96.89 0.01 0.03 +w=zip(100)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 +w=zip(10000)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 19.50 37.19 0.01 0.01 +w=uni(10000)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 +w=zip(100)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 15.84 28.80 0.00 0.01 +w=uni(10000)/k=uni(10000)/s=uni(1)/s(%)=0/10000 +w=zip(10000)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 6.44 13.72 0.02 0.05 +w=uni(100)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 +w=zip(10000)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 4.39 15.13 0.00 0.01 +w=zip(100)/k=uni(10000)/s=uni(1)/s(%)=0/10000 +w=zip(100)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 8.39 25.77 0.00 0.01 +w=zip(100)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 +w=uni(100)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 9.83 28.70 0.00 0.01 +w=uni(100)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 +w=uni(10000)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 4.72 16.01 0.01 0.03 +w=zip(10000)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 +w=zip(10000)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 61.38 68.35 0.00 0.01 +w=uni(10000)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 +w=uni(100)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 24.80 39.37 0.00 0.01 +w=uni(10000)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 +w=zip(10000)/k=zip(10000)/s=uni(1)/s(%)=0/10000 +mixed_requests(2) 0.00 22.69 33.55 0.01 0.03 +w=zip(100)/k=zip(10000)/s=uni(1)/s(%)=0/10000 +w=zip(10000)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 30.61 44.26 0.00 0.01 +w=zip(100)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 +w=uni(100)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 +mixed_requests(2) 0.00 16.24 33.52 0.00 0.01 +w=zip(10000)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 +w=uni(10000)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 33.70 43.45 0.01 0.02 +w=uni(10000)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 +w=zip(100)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 29.09 41.81 0.00 0.01 +w=zip(100)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 +w=uni(100)/k=zip(10000)/s=uni(1)/s(%)=0/10000 +mixed_requests(2) 0.00 6.18 22.54 0.01 0.04 +w=zip(10000)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 +w=uni(100)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 17.38 28.91 0.00 0.01 +w=uni(10000)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 +w=uni(100)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 +mixed_requests(2) 0.00 9.19 14.37 0.00 0.02 +w=uni(100)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 +w=uni(10000)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 85.88 90.20 0.02 0.07 +w=zip(10000)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 +w=uni(100)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 25.41 50.54 0.03 0.09 +w=zip(10000)/k=uni(10000)/s=uni(1)/s(%)=0/10000 +w=zip(100)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 6.04 16.76 0.00 0.01 +w=uni(100)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 +w=zip(100)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 61.46 70.67 0.01 0.02 +w=uni(10000)/k=zip(10000)/s=uni(1)/s(%)=0/10000 +w=zip(10000)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 92.36 96.84 0.02 0.04 +w=zip(100)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 +w=zip(10000)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 70.34 76.14 0.02 0.07 +w=zip(10000)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 +w=zip(100)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 +mixed_requests(2) 0.00 5.56 23.88 0.00 0.00 +w=uni(10000)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 +w=uni(10000)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 6.57 17.71 0.01 0.02 +w=uni(10000)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 +w=zip(100)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 6.10 18.17 0.00 0.01 +w=uni(100)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 +w=uni(100)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 4.82 19.40 0.00 0.00 +w=uni(10000)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 +w=uni(10000)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 7.19 19.78 0.02 0.08 +w=uni(100)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 +w=zip(10000)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 58.71 76.50 0.01 0.04 +w=uni(100)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 +w=zip(10000)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 7.10 22.10 0.01 0.02 +w=zip(100)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 +w=zip(100)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 62.81 70.33 0.00 0.01 +w=uni(10000)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 +w=zip(10000)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 5.59 24.12 0.00 0.02 +w=zip(100)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 +w=zip(100)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 21.42 24.93 0.01 0.02 +w=uni(10000)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 +w=uni(100)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 19.11 23.27 0.01 0.03 +w=zip(10000)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 +w=zip(100)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 +mixed_requests(2) 0.00 90.57 98.25 0.00 0.01 +w=uni(10000)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 +w=uni(100)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 diff --git a/pkg/kv/kvserver/split/testdata/unweighted_finder b/pkg/kv/kvserver/split/testdata/unweighted_finder new file mode 100644 index 000000000000..5f88be57d94d --- /dev/null +++ b/pkg/kv/kvserver/split/testdata/unweighted_finder @@ -0,0 +1,10 @@ +requests key_dist=zipfian key_max=10000000000 span_dist=uniform span_max=1000 weight_dist=uniform weight_max=1 range_request_percent=95 request_count=13000 +---- + +finder weighted=false +---- + +eval seed=2022 iterations=100 +---- +description no_key(%) avg_diff(%) max_diff(%) avg_optimal_diff(%) max_optimal_diff(%) +w=uni(1)/k=zip(10000000000)/s=uni(1000)/s(%)=95/13000 1.00 4.65 26.64 0.00 0.01 diff --git a/pkg/kv/kvserver/split/testdata/weighted_finder b/pkg/kv/kvserver/split/testdata/weighted_finder new file mode 100644 index 000000000000..bd651033a453 --- /dev/null +++ b/pkg/kv/kvserver/split/testdata/weighted_finder @@ -0,0 +1,49 @@ +# Test the weighted finder with a set of known request configurations. +finder weighted=true +---- + +requests key_dist=zipfian key_max=10000000000 span_dist=uniform span_max=1000 weight_dist=uniform weight_max=10 range_request_percent=95 request_count=10000 +---- + +requests key_dist=zipfian key_max=100000 span_dist=uniform span_max=1000 weight_dist=uniform weight_max=10 range_request_percent=95 request_count=10000 +---- + +requests key_dist=zipfian key_max=1000 span_dist=uniform span_max=100 weight_dist=uniform weight_max=10 range_request_percent=95 request_count=10000 +---- + +requests key_dist=zipfian key_max=1000 span_dist=uniform span_max=1000 weight_dist=uniform weight_max=10 range_request_percent=0 request_count=10000 +---- + +requests key_dist=zipfian key_max=10000000000 span_dist=uniform span_max=1000 weight_dist=zipfian weight_max=1000 range_request_percent=50 request_count=10000 +---- + +eval seed=2022 iterations=100 all=true mix=sequential mix_count=5 +---- +description no_key(%) avg_diff(%) max_diff(%) avg_optimal_diff(%) max_optimal_diff(%) +mixed_requests(5) 3.00 10.87 29.92 0.01 0.05 +w=uni(10)/k=zip(1000)/s=uni(100)/s(%)=95/10000 +w=uni(10)/k=zip(1000)/s=uni(1000)/s(%)=0/10000 +w=uni(10)/k=zip(10000000000)/s=uni(1000)/s(%)=95/10000 +w=uni(10)/k=zip(100000)/s=uni(1000)/s(%)=95/10000 +w=zip(1000)/k=zip(10000000000)/s=uni(1000)/s(%)=50/10000 + +eval seed=42 iterations=100 all=true mix=sequential mix_count=2 +---- +description no_key(%) avg_diff(%) max_diff(%) avg_optimal_diff(%) max_optimal_diff(%) +mixed_requests(2) 3.00 36.26 62.78 0.00 0.01 +w=uni(10)/k=zip(1000)/s=uni(100)/s(%)=95/10000 +w=uni(10)/k=zip(10000000000)/s=uni(1000)/s(%)=95/10000 +mixed_requests(2) 0.00 6.22 26.87 0.01 0.05 +w=uni(10)/k=zip(100000)/s=uni(1000)/s(%)=95/10000 +w=zip(1000)/k=zip(10000000000)/s=uni(1000)/s(%)=50/10000 +w=uni(10)/k=zip(1000)/s=uni(1000)/s(%)=0/10000 2.00 4.90 23.89 0.01 0.02 + +# Next test out a config with only one key. There shouldn't be any splits +# possible. +requests key_dist=zipfian key_max=1 span_dist=uniform span_max=1 weight_dist=uniform weight_max=10 range_request_percent=0 request_count=10000 +---- + +eval seed=42 iterations=100 +---- +description no_key(%) avg_diff(%) max_diff(%) avg_optimal_diff(%) max_optimal_diff(%) +w=uni(10)/k=zip(1)/s=uni(1)/s(%)=0/10000 100.00 NaN 0.00 NaN 0.00 From 452984332920cde1f0f4c04f256f689b2a6a32d0 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Wed, 26 Apr 2023 20:07:13 +0000 Subject: [PATCH 2/6] split: use weight in decider Previously the load-based split decider would pass in 1 to the finder, whether it were the weighted or unweighted variation. This defeats the purpose of using the weighted finder. This commit passes in the weight. Fixes: cockroachdb#102132 Release note: None --- pkg/kv/kvserver/split/decider.go | 2 +- .../split/testdata/cpu_decider_cartesian | 80 +++++++++---------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pkg/kv/kvserver/split/decider.go b/pkg/kv/kvserver/split/decider.go index c321623684d1..ac0998257442 100644 --- a/pkg/kv/kvserver/split/decider.go +++ b/pkg/kv/kvserver/split/decider.go @@ -229,7 +229,7 @@ func (d *Decider) recordLocked( if d.mu.splitFinder != nil && n != 0 { s := span() if s.Key != nil { - d.mu.splitFinder.Record(span(), 1) + d.mu.splitFinder.Record(span(), float64(n)) } if d.mu.splitFinder.Ready(now) { if d.mu.splitFinder.Key() != nil { diff --git a/pkg/kv/kvserver/split/testdata/cpu_decider_cartesian b/pkg/kv/kvserver/split/testdata/cpu_decider_cartesian index 1799811f27f6..c9cc81c583d6 100644 --- a/pkg/kv/kvserver/split/testdata/cpu_decider_cartesian +++ b/pkg/kv/kvserver/split/testdata/cpu_decider_cartesian @@ -9,123 +9,123 @@ decider duration=100 retention=200 objective=cpu threshold=1000 eval seed=42 iterations=20 cartesian=true mix=perm mix_count=2 ---- description no_key(%) avg_diff(%) max_diff(%) avg_optimal_diff(%) max_optimal_diff(%) -mixed_requests(2) 0.00 91.53 97.71 0.01 0.01 +mixed_requests(2) 0.00 3.71 13.23 0.01 0.02 w=uni(10000)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 w=zip(100)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 18.46 25.86 0.03 0.09 +mixed_requests(2) 0.00 7.97 20.82 0.02 0.05 w=uni(100)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 w=zip(10000)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 6.49 18.32 0.02 0.08 +mixed_requests(2) 0.00 5.52 12.77 0.02 0.08 w=zip(10000)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 w=uni(100)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 5.56 17.30 0.00 0.01 +mixed_requests(2) 0.00 4.74 10.57 0.01 0.01 w=uni(10000)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 w=uni(100)/k=uni(10000)/s=uni(1)/s(%)=0/10000 -mixed_requests(2) 0.00 90.54 96.89 0.01 0.03 +mixed_requests(2) 0.00 5.98 19.81 0.01 0.04 w=zip(100)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 w=zip(10000)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 19.50 37.19 0.01 0.01 +mixed_requests(2) 0.00 4.97 15.74 0.01 0.02 w=uni(10000)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 w=zip(100)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 15.84 28.80 0.00 0.01 +mixed_requests(2) 0.00 5.06 21.37 0.01 0.01 w=uni(10000)/k=uni(10000)/s=uni(1)/s(%)=0/10000 w=zip(10000)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 6.44 13.72 0.02 0.05 +mixed_requests(2) 0.00 3.69 19.97 0.03 0.08 w=uni(100)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 w=zip(10000)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 4.39 15.13 0.00 0.01 +mixed_requests(2) 0.00 6.36 15.63 0.01 0.02 w=zip(100)/k=uni(10000)/s=uni(1)/s(%)=0/10000 w=zip(100)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 8.39 25.77 0.00 0.01 +mixed_requests(2) 0.00 4.11 16.75 0.00 0.01 w=zip(100)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 w=uni(100)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 9.83 28.70 0.00 0.01 +mixed_requests(2) 0.00 4.80 17.67 0.00 0.01 w=uni(100)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 w=uni(10000)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 4.72 16.01 0.01 0.03 +mixed_requests(2) 0.00 5.33 28.11 0.01 0.03 w=zip(10000)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 w=zip(10000)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 61.38 68.35 0.00 0.01 +mixed_requests(2) 0.00 4.89 14.73 0.00 0.01 w=uni(10000)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 w=uni(100)/k=uni(10000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 24.80 39.37 0.00 0.01 +mixed_requests(2) 0.00 5.32 12.46 0.00 0.01 w=uni(10000)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 w=zip(10000)/k=zip(10000)/s=uni(1)/s(%)=0/10000 -mixed_requests(2) 0.00 22.69 33.55 0.01 0.03 +mixed_requests(2) 0.00 5.02 14.09 0.01 0.04 w=zip(100)/k=zip(10000)/s=uni(1)/s(%)=0/10000 w=zip(10000)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 30.61 44.26 0.00 0.01 +mixed_requests(2) 0.00 6.14 18.16 0.00 0.01 w=zip(100)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 w=uni(100)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 -mixed_requests(2) 0.00 16.24 33.52 0.00 0.01 +mixed_requests(2) 0.00 3.92 14.73 0.00 0.01 w=zip(10000)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 w=uni(10000)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 33.70 43.45 0.01 0.02 +mixed_requests(2) 0.00 5.83 22.03 0.01 0.02 w=uni(10000)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 w=zip(100)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 29.09 41.81 0.00 0.01 +mixed_requests(2) 0.00 4.60 13.13 0.01 0.01 w=zip(100)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 w=uni(100)/k=zip(10000)/s=uni(1)/s(%)=0/10000 -mixed_requests(2) 0.00 6.18 22.54 0.01 0.04 +mixed_requests(2) 0.00 5.71 22.33 0.01 0.06 w=zip(10000)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 w=uni(100)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 17.38 28.91 0.00 0.01 +mixed_requests(2) 0.00 5.13 16.86 0.01 0.02 w=uni(10000)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 w=uni(100)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 -mixed_requests(2) 0.00 9.19 14.37 0.00 0.02 +mixed_requests(2) 0.00 3.50 8.96 0.00 0.01 w=uni(100)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 w=uni(10000)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 85.88 90.20 0.02 0.07 +mixed_requests(2) 0.00 4.48 14.54 0.02 0.07 w=zip(10000)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 w=uni(100)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 25.41 50.54 0.03 0.09 +mixed_requests(2) 0.00 6.88 22.68 0.02 0.07 w=zip(10000)/k=uni(10000)/s=uni(1)/s(%)=0/10000 w=zip(100)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 6.04 16.76 0.00 0.01 +mixed_requests(2) 0.00 4.80 24.10 0.00 0.00 w=uni(100)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 w=zip(100)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 61.46 70.67 0.01 0.02 +mixed_requests(2) 0.00 6.29 24.99 0.00 0.02 w=uni(10000)/k=zip(10000)/s=uni(1)/s(%)=0/10000 w=zip(10000)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 92.36 96.84 0.02 0.04 +mixed_requests(2) 0.00 6.39 13.28 0.02 0.07 w=zip(100)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 w=zip(10000)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 70.34 76.14 0.02 0.07 +mixed_requests(2) 0.00 4.08 11.02 0.03 0.08 w=zip(10000)/k=zip(1000000)/s=uni(1000)/s(%)=20/10000 w=zip(100)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 -mixed_requests(2) 0.00 5.56 23.88 0.00 0.00 +mixed_requests(2) 0.00 4.84 13.76 0.00 0.01 w=uni(10000)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 w=uni(10000)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 6.57 17.71 0.01 0.02 +mixed_requests(2) 0.00 4.08 8.81 0.01 0.02 w=uni(10000)/k=uni(1000000)/s=uni(1)/s(%)=0/10000 w=zip(100)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 6.10 18.17 0.00 0.01 +mixed_requests(2) 0.00 4.89 12.20 0.00 0.01 w=uni(100)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 w=uni(100)/k=uni(1000000)/s=uni(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 4.82 19.40 0.00 0.00 +mixed_requests(2) 0.00 5.03 16.77 0.00 0.00 w=uni(10000)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 w=uni(10000)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 7.19 19.78 0.02 0.08 +mixed_requests(2) 0.00 5.07 15.26 0.02 0.06 w=uni(100)/k=zip(1000000)/s=zip(1000)/s(%)=95/10000 w=zip(10000)/k=zip(1000000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 58.71 76.50 0.01 0.04 +mixed_requests(2) 0.00 5.41 17.41 0.01 0.04 w=uni(100)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 w=zip(10000)/k=uni(10000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 7.10 22.10 0.01 0.02 +mixed_requests(2) 0.00 4.22 12.22 0.01 0.02 w=zip(100)/k=zip(10000)/s=uni(1000)/s(%)=20/10000 w=zip(100)/k=zip(10000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 62.81 70.33 0.00 0.01 +mixed_requests(2) 0.00 3.26 13.03 0.00 0.01 w=uni(10000)/k=uni(1000000)/s=zip(1000)/s(%)=95/10000 w=zip(10000)/k=zip(10000)/s=zip(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 5.59 24.12 0.00 0.02 +mixed_requests(2) 0.00 4.11 14.78 0.00 0.01 w=zip(100)/k=zip(10000)/s=uni(1000)/s(%)=95/10000 w=zip(100)/k=uni(1000000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 21.42 24.93 0.01 0.02 +mixed_requests(2) 0.00 6.28 17.10 0.01 0.01 w=uni(10000)/k=zip(1000000)/s=uni(1)/s(%)=0/10000 w=uni(100)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 -mixed_requests(2) 0.00 19.11 23.27 0.01 0.03 +mixed_requests(2) 0.00 5.17 13.51 0.01 0.03 w=zip(10000)/k=zip(1000000)/s=uni(1000)/s(%)=95/10000 w=zip(100)/k=uni(10000)/s=zip(1000)/s(%)=20/10000 -mixed_requests(2) 0.00 90.57 98.25 0.00 0.01 +mixed_requests(2) 0.00 4.23 20.33 0.00 0.01 w=uni(10000)/k=uni(10000)/s=uni(1000)/s(%)=95/10000 w=uni(100)/k=uni(1000000)/s=uni(1000)/s(%)=95/10000 From 2083f7b60c7a37da23d0e3936c404da9b07d4708 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Mon, 24 Apr 2023 23:43:49 +0000 Subject: [PATCH 3/6] split: only retain safe sql key samples Previously, the weighted load split finder could retain samples for any key that was a start key of a request span. This was problematic as the split key which is used, always has user column suffixes removed by calling `keys.EnsureSafeSplitKey` - meaning that it was possible for every sampled key to refer to the same key after this call. We could convert every incoming key into the safe split key, however this doesn't seem like a worthwhile trade off the additional overhead - as every request touches this hot path. This patch updates the sampling logic to always store the safe split key when a sampled key is replaced or added. The samples may still contain only a single key, however now it is much more likely that we will log due imbalance (all requests to the RHS) and bump the split finder no split key metrics. Release note: None --- pkg/kv/kvserver/split/weighted_finder.go | 18 ++++++++++++++++++ pkg/kv/kvserver/split/weighted_finder_test.go | 15 +++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/pkg/kv/kvserver/split/weighted_finder.go b/pkg/kv/kvserver/split/weighted_finder.go index 773c548e613f..a1a66e354a39 100644 --- a/pkg/kv/kvserver/split/weighted_finder.go +++ b/pkg/kv/kvserver/split/weighted_finder.go @@ -16,6 +16,7 @@ import ( "sort" "time" + "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" ) @@ -123,6 +124,23 @@ func (f *WeightedFinder) record(key roachpb.Key, weight float64) { idx = f.randSource.Intn(splitKeySampleSize) } + // We only wish to retain safe split keys as samples, as they are the split + // keys that will eventually be returned from Key(). If instead we kept every + // key, it is possible for all sample keys to map to the same split key + // implicitly with column families. Note this doesn't stop every sample being + // the same key, however it will cause no split key logging and bump metrics. + // TODO(kvoli): When the single key situation arises, we should backoff + // attempting to split. There is a fixed overhead on the hotpath when the + // finder is active. + if safeKey, err := keys.EnsureSafeSplitKey(key); err == nil { + key = safeKey + } else { + // If the key is not a safe split key, instead ignore it and don't bump any + // counters. This biases the algorithm slightly, as keys which would be + // invalid are not sampled, nor their impact recorded if they reach here. + return + } + // Note we always use the start key of the span. We could // take the average of the byte slices, but that seems // unnecessarily complex for practical usage. diff --git a/pkg/kv/kvserver/split/weighted_finder_test.go b/pkg/kv/kvserver/split/weighted_finder_test.go index 6db32a6bae3f..5a86b79c052d 100644 --- a/pkg/kv/kvserver/split/weighted_finder_test.go +++ b/pkg/kv/kvserver/split/weighted_finder_test.go @@ -191,6 +191,17 @@ func TestSplitWeightedFinderRecorder(t *testing.T) { stopper := stop.NewStopper() defer stopper.Stop(context.Background()) + colKey := func(prefix roachpb.Key) roachpb.Key { + return keys.MakeFamilyKey(prefix, 9) + } + + colFamSpan := func(span roachpb.Span) roachpb.Span { + return roachpb.Span{ + Key: colKey(span.Key), + EndKey: colKey(span.EndKey), + } + } + const ReservoirKeyOffset = 1000 // Test recording a key query before the reservoir is full. @@ -284,12 +295,16 @@ func TestSplitWeightedFinderRecorder(t *testing.T) { }{ // Test recording a key query before the reservoir is full. {basicSpan, basicWeight, WFLargestRandSource{}, 0, basicReservoir, expectedBasicReservoir}, + {colFamSpan(basicSpan), basicWeight, WFLargestRandSource{}, 0, basicReservoir, expectedBasicReservoir}, // Test recording a key query after the reservoir is full with replacement. {replacementSpan, replacementWeight, ZeroRandSource{}, splitKeySampleSize + 1, replacementReservoir, expectedReplacementReservoir}, + {colFamSpan(replacementSpan), replacementWeight, ZeroRandSource{}, splitKeySampleSize + 1, replacementReservoir, expectedReplacementReservoir}, // Test recording a key query after the reservoir is full without replacement. {fullSpan, fullWeight, WFLargestRandSource{}, splitKeySampleSize + 1, fullReservoir, expectedFullReservoir}, + {colFamSpan(fullSpan), fullWeight, WFLargestRandSource{}, splitKeySampleSize + 1, fullReservoir, expectedFullReservoir}, // Test recording a spanning query. {spanningSpan, spanningWeight, WFLargestRandSource{}, splitKeySampleSize + 1, spanningReservoir, expectedSpanningReservoir}, + {colFamSpan(spanningSpan), spanningWeight, WFLargestRandSource{}, splitKeySampleSize + 1, spanningReservoir, expectedSpanningReservoir}, } for i, test := range testCases { From 0b4d0aa7a7ef8e2541a42a2610d6d84ca0a87950 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Mon, 24 Apr 2023 21:48:01 +0000 Subject: [PATCH 4/6] roachtest: add cpu split threshold to split tests Previously, only setting a QPS split threshold and objective was possible in the `splits/load` roachtests. This commit enables setting cpu as an objective and providing a threshold. Part of #97540 Release note: None --- pkg/cmd/roachtest/tests/split.go | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/pkg/cmd/roachtest/tests/split.go b/pkg/cmd/roachtest/tests/split.go index ef1a2cb524cb..8b2f41cbd5a0 100644 --- a/pkg/cmd/roachtest/tests/split.go +++ b/pkg/cmd/roachtest/tests/split.go @@ -38,6 +38,7 @@ type splitParams struct { readPercent int // % of queries that are read queries. spanPercent int // % of queries that query all the rows. qpsThreshold int // QPS Threshold for load based splitting. + cpuThreshold time.Duration // CPU Threshold for load based splitting. minimumRanges int // Minimum number of ranges expected at the end. maximumRanges int // Maximum number of ranges expected at the end. sequential bool // Sequential distribution. @@ -149,9 +150,29 @@ func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params s // TODO(kvoli): Add load split tests which use CPU, similar to the current // QPS ones. Tracked by #97540. - t.Status("setting split objective to QPS") - if err := setLoadBasedRebalancingObjective(ctx, db, "qps"); err != nil { - return err + // + // Set the objective to QPS or CPU and update the load split threshold + // appropriately. + if params.qpsThreshold > 0 { + t.Status("setting split objective to QPS with threshold %d", params.qpsThreshold) + if err := setLoadBasedRebalancingObjective(ctx, db, "qps"); err != nil { + return err + } + if _, err := db.ExecContext(ctx, fmt.Sprintf("SET CLUSTER SETTING kv.range_split.load_qps_threshold = %d", + params.qpsThreshold)); err != nil { + return err + } + } else if params.cpuThreshold > 0 { + t.Status("setting split objective to CPU with threshold %s", params.cpuThreshold) + if err := setLoadBasedRebalancingObjective(ctx, db, "cpu"); err != nil { + return err + } + if _, err := db.ExecContext(ctx, fmt.Sprintf("SET CLUSTER SETTING kv.range_split.load_cpu_threshold = '%s'", + params.cpuThreshold)); err != nil { + return err + } + } else { + t.Fatal("no CPU or QPS split threshold set") } t.Status("increasing range_max_bytes") @@ -192,11 +213,6 @@ func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params s return errors.Errorf("kv.kv table split over multiple ranges.") } - // Set the QPS threshold for load based splitting before turning it on. - if _, err := db.ExecContext(ctx, fmt.Sprintf("SET CLUSTER SETTING kv.range_split.load_qps_threshold = %d", - params.qpsThreshold)); err != nil { - return err - } t.Status("enable load based splitting") if _, err := db.ExecContext(ctx, `SET CLUSTER SETTING kv.range_split.by_load_enabled = true`); err != nil { return err From e9effbb25d5eb786955a0ca932a8dbdd85baecac Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Mon, 24 Apr 2023 22:16:13 +0000 Subject: [PATCH 5/6] roachtest: separate out workload in splits/load This commit separates out the workload from the verification parameters in the `splits/load` roachtests. This is done so that different workloads, not just KV can be run in the load split test runner. Part of: #97540 Release note: None --- pkg/cmd/roachtest/tests/split.go | 176 +++++++++++++++++++------------ 1 file changed, 106 insertions(+), 70 deletions(-) diff --git a/pkg/cmd/roachtest/tests/split.go b/pkg/cmd/roachtest/tests/split.go index 8b2f41cbd5a0..3a90f717399c 100644 --- a/pkg/cmd/roachtest/tests/split.go +++ b/pkg/cmd/roachtest/tests/split.go @@ -32,17 +32,65 @@ import ( "github.com/stretchr/testify/require" ) +type splitLoad interface { + // init initializes the split workload. + init(context.Context, test.Test, cluster.Cluster) error + // run starts the split workload. + run(context.Context, test.Test, cluster.Cluster) error + // rangeCount returns the range count for the split workload ranges. + rangeCount(*gosql.DB) (int, error) +} + +type kvSplitLoad struct { + // concurrency is the number of concurrent workers. + concurrency int + // readPercent is the % of queries that are read queries. + readPercent int + // spanPercent is the % of queries that query all the rows. + spanPercent int + // sequential indicates the kv workload will use a sequential distribution. + sequential bool + // waitDuration is the duration the workload should run for. + waitDuration time.Duration +} + +func (ksl kvSplitLoad) init(ctx context.Context, t test.Test, c cluster.Cluster) error { + t.Status("running uniform kv workload") + return c.RunE(ctx, c.Node(1), fmt.Sprintf("./workload init kv {pgurl:1-%d}", c.Spec().NodeCount)) +} + +func (ksl kvSplitLoad) rangeCount(db *gosql.DB) (int, error) { + return rangeCountFrom("kv.kv", db) +} + +func (ksl kvSplitLoad) run(ctx context.Context, t test.Test, c cluster.Cluster) error { + var extraFlags string + if ksl.sequential { + extraFlags += "--sequential" + } + return c.RunE(ctx, c.Node(1), fmt.Sprintf("./workload run kv "+ + "--init --concurrency=%d --read-percent=%d --span-percent=%d %s {pgurl:1-%d} --duration='%s'", + ksl.concurrency, ksl.readPercent, ksl.spanPercent, extraFlags, c.Spec().NodeCount, + ksl.waitDuration.String())) +} + +func rangeCountFrom(from string, db *gosql.DB) (int, error) { + var ranges int + q := fmt.Sprintf("SELECT count(*) FROM [SHOW RANGES FROM TABLE %s]", + from) + if err := db.QueryRow(q).Scan(&ranges); err != nil { + return 0, err + } + return ranges, nil +} + type splitParams struct { + load splitLoad maxSize int // The maximum size a range is allowed to be. - concurrency int // Number of concurrent workers. - readPercent int // % of queries that are read queries. - spanPercent int // % of queries that query all the rows. qpsThreshold int // QPS Threshold for load based splitting. cpuThreshold time.Duration // CPU Threshold for load based splitting. minimumRanges int // Minimum number of ranges expected at the end. maximumRanges int // Maximum number of ranges expected at the end. - sequential bool // Sequential distribution. - waitDuration time.Duration // Duration the workload should run for. } func registerLoadSplits(r registry.Registry) { @@ -58,37 +106,39 @@ func registerLoadSplits(r registry.Registry) { expSplits := 10 runLoadSplits(ctx, t, c, splitParams{ maxSize: 10 << 30, // 10 GB - concurrency: 64, // 64 concurrent workers - readPercent: 95, // 95% reads qpsThreshold: 100, // 100 queries per second minimumRanges: expSplits + 1, // Expected Splits + 1 maximumRanges: math.MaxInt32, // We're only checking for minimum. - // The calculation of the wait duration is as follows: - // - // Each split requires at least `split.RecordDurationThreshold` seconds to record - // keys in a range. So in the kv default distribution, if we make the assumption - // that all load will be uniform across the splits AND that the QPS threshold is - // still exceeded for all the splits as the number of splits we're targeting is - // "low" - we expect that for `expSplits` splits, it will require: - // - // Minimum Duration For a Split * log2(expSplits) seconds - // - // We also add an extra expSplits second(s) for the overhead of creating each one. - // If the number of expected splits is increased, this calculation will hold - // for uniform distribution as long as the QPS threshold is continually exceeded - // even with the expected number of splits. This puts a bound on how high the - // `expSplits` value can go. - // Add 1s for each split for the overhead of the splitting process. - // waitDuration: time.Duration(int64(math.Ceil(math.Ceil(math.Log2(float64(expSplits)))* - // float64((split.RecordDurationThreshold/time.Second))))+int64(expSplits)) * time.Second, - // - // NB: the above has proven flaky. Just use a fixed duration - // that we think should be good enough. For example, for five - // expected splits we get ~35s, for ten ~50s, and for 20 ~1m10s. - // These are all pretty short, so any random abnormality will mess - // things up. - waitDuration: 10 * time.Minute, - }) + + load: kvSplitLoad{ + concurrency: 64, // 64 concurrent workers + readPercent: 95, // 95% reads + // The calculation of the wait duration is as follows: + // + // Each split requires at least `split.RecordDurationThreshold` seconds to record + // keys in a range. So in the kv default distribution, if we make the assumption + // that all load will be uniform across the splits AND that the QPS threshold is + // still exceeded for all the splits as the number of splits we're targeting is + // "low" - we expect that for `expSplits` splits, it will require: + // + // Minimum Duration For a Split * log2(expSplits) seconds + // + // We also add an extra expSplits second(s) for the overhead of creating each one. + // If the number of expected splits is increased, this calculation will hold + // for uniform distribution as long as the QPS threshold is continually exceeded + // even with the expected number of splits. This puts a bound on how high the + // `expSplits` value can go. + // Add 1s for each split for the overhead of the splitting process. + // waitDuration: time.Duration(int64(math.Ceil(math.Ceil(math.Log2(float64(expSplits)))* + // float64((split.RecordDurationThreshold/time.Second))))+int64(expSplits)) * time.Second, + // + // NB: the above has proven flaky. Just use a fixed duration + // that we think should be good enough. For example, for five + // expected splits we get ~35s, for ten ~50s, and for 20 ~1m10s. + // These are all pretty short, so any random abnormality will mess + // things up. + waitDuration: 10 * time.Minute, + }}) }, }) r.Add(registry.TestSpec{ @@ -98,17 +148,18 @@ func registerLoadSplits(r registry.Registry) { Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runLoadSplits(ctx, t, c, splitParams{ maxSize: 10 << 30, // 10 GB - concurrency: 64, // 64 concurrent workers - readPercent: 0, // 0% reads qpsThreshold: 100, // 100 queries per second minimumRanges: 1, // We expect no splits so require only 1 range. // We expect no splits so require only 1 range. However, in practice we // sometimes see a split or two early in, presumably when the sampling // gets lucky. maximumRanges: 3, - sequential: true, - waitDuration: 60 * time.Second, - }) + load: kvSplitLoad{ + concurrency: 64, // 64 concurrent workers + readPercent: 0, // 0% reads + sequential: true, + waitDuration: 60 * time.Second, + }}) }, }) r.Add(registry.TestSpec{ @@ -118,14 +169,15 @@ func registerLoadSplits(r registry.Registry) { Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runLoadSplits(ctx, t, c, splitParams{ maxSize: 10 << 30, // 10 GB - concurrency: 64, // 64 concurrent workers - readPercent: 0, // 0% reads - spanPercent: 95, // 95% spanning queries qpsThreshold: 100, // 100 queries per second minimumRanges: 1, // We expect no splits so require only 1 range. maximumRanges: 1, // We expect no splits so require only 1 range. - waitDuration: 60 * time.Second, - }) + load: kvSplitLoad{ + concurrency: 64, // 64 concurrent workers + readPercent: 0, // 0% reads + spanPercent: 95, // 95% spanning queries + waitDuration: 60 * time.Second, + }}) }, }) } @@ -190,26 +242,13 @@ func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params s // range unless split by load. setRangeMaxBytes(params.maxSize) - t.Status("running uniform kv workload") - c.Run(ctx, c.Node(1), fmt.Sprintf("./workload init kv {pgurl:1-%d}", c.Spec().NodeCount)) + // Init the split workload. + if err := params.load.init(ctx, t, c); err != nil { + t.Fatal(err) + } t.Status("checking initial range count") - rangeCount := func() int { - var ranges int - const q = "SELECT count(*) FROM [SHOW RANGES FROM TABLE kv.kv]" - if err := db.QueryRow(q).Scan(&ranges); err != nil { - // TODO(rafi): Remove experimental_ranges query once we stop testing - // 19.1 or earlier. - if strings.Contains(err.Error(), "syntax error at or near \"ranges\"") { - err = db.QueryRow("SELECT count(*) FROM [SHOW EXPERIMENTAL_RANGES FROM TABLE kv.kv]").Scan(&ranges) - } - if err != nil { - t.Fatalf("failed to get range count: %v", err) - } - } - return ranges - } - if rc := rangeCount(); rc != 1 { + if rc, _ := params.load.rangeCount(db); rc != 1 { return errors.Errorf("kv.kv table split over multiple ranges.") } @@ -217,17 +256,14 @@ func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params s if _, err := db.ExecContext(ctx, `SET CLUSTER SETTING kv.range_split.by_load_enabled = true`); err != nil { return err } - var extraFlags string - if params.sequential { - extraFlags += "--sequential" - } - c.Run(ctx, c.Node(1), fmt.Sprintf("./workload run kv "+ - "--init --concurrency=%d --read-percent=%d --span-percent=%d %s {pgurl:1-%d} --duration='%s'", - params.concurrency, params.readPercent, params.spanPercent, extraFlags, c.Spec().NodeCount, - params.waitDuration.String())) + if err := params.load.run(ctx, t, c); err != nil { + return err + } t.Status("waiting for splits") - if rc := rangeCount(); rc < params.minimumRanges || rc > params.maximumRanges { + if rc, err := params.load.rangeCount(db); err != nil { + t.Fatal(err) + } else if rc < params.minimumRanges || rc > params.maximumRanges { return errors.Errorf("kv.kv has %d ranges, expected between %d and %d splits", rc, params.minimumRanges, params.maximumRanges) } From 5df42dc451453b962fe0b8ab1078eb65ed6ab456 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Mon, 24 Apr 2023 22:58:08 +0000 Subject: [PATCH 6/6] roachtest: add cpu load split tests There were no roachtests which used the new CPU load based splitter. This commit adds similar tests as those that exist for QPS: - spanning - uniform - sequential In addition to select YCSB tests which have a column family schema and zipfian/latest distributions. It is a known issue that the YCSB workloads will commonly return a start key due to column families when splitting a range with a single hot row #102136. - YCSB/A - YCSB/B - YCSB/D - YCSB/E Both the minimum and maximimum number of ranges after some time is asserted on. For YCSB and KV uniform, the period is 10 minutes. For KV spanning and sequential, the period is 60 seconds. Unlike QPS splitting, which will not attempt to split Spanning requests, CPU splitting will attempt to split ranges with higher CPU than the threshold, as this does not result in signal amplification unlike QPS - which doubles the QPS on each split of span request heavy ranges. Resolves: #97540 Release note: None --- pkg/cmd/roachtest/tests/split.go | 254 ++++++++++++++++++++++++++++--- 1 file changed, 231 insertions(+), 23 deletions(-) diff --git a/pkg/cmd/roachtest/tests/split.go b/pkg/cmd/roachtest/tests/split.go index 3a90f717399c..d4a7ec5941f3 100644 --- a/pkg/cmd/roachtest/tests/split.go +++ b/pkg/cmd/roachtest/tests/split.go @@ -50,13 +50,15 @@ type kvSplitLoad struct { spanPercent int // sequential indicates the kv workload will use a sequential distribution. sequential bool + // blockSize controls the size of writes to the kv table. + blockSize int // waitDuration is the duration the workload should run for. waitDuration time.Duration } func (ksl kvSplitLoad) init(ctx context.Context, t test.Test, c cluster.Cluster) error { t.Status("running uniform kv workload") - return c.RunE(ctx, c.Node(1), fmt.Sprintf("./workload init kv {pgurl:1-%d}", c.Spec().NodeCount)) + return c.RunE(ctx, c.Node(c.Spec().NodeCount), fmt.Sprintf("./workload init kv {pgurl:1-%d}", c.Spec().NodeCount-1)) } func (ksl kvSplitLoad) rangeCount(db *gosql.DB) (int, error) { @@ -66,14 +68,60 @@ func (ksl kvSplitLoad) rangeCount(db *gosql.DB) (int, error) { func (ksl kvSplitLoad) run(ctx context.Context, t test.Test, c cluster.Cluster) error { var extraFlags string if ksl.sequential { - extraFlags += "--sequential" + extraFlags += "--sequential " } - return c.RunE(ctx, c.Node(1), fmt.Sprintf("./workload run kv "+ + if ksl.blockSize != 0 { + extraFlags += fmt.Sprintf("--min-block-bytes=%d --max-block-bytes=%d ", + ksl.blockSize, ksl.blockSize) + } + return c.RunE(ctx, c.Node(c.Spec().NodeCount), fmt.Sprintf("./workload run kv "+ "--init --concurrency=%d --read-percent=%d --span-percent=%d %s {pgurl:1-%d} --duration='%s'", - ksl.concurrency, ksl.readPercent, ksl.spanPercent, extraFlags, c.Spec().NodeCount, + ksl.concurrency, ksl.readPercent, ksl.spanPercent, extraFlags, c.Spec().NodeCount-1, ksl.waitDuration.String())) } +type ycsbSplitLoad struct { + // workload is the YCSB workload letter e.g. a, b, ..., f. + workload string + // concurrency is the number of concurrent workers. + concurrency int + // hashed determines whether the inserted keys are hashed. + hashed bool + // insertCount is the number of records to pre-load into the user table. + insertCount int + // waitDuration is the duration the workload should run for. + waitDuration time.Duration +} + +func (ysl ycsbSplitLoad) init(ctx context.Context, t test.Test, c cluster.Cluster) error { + t.Status("running ycsb workload ", ysl.workload) + extraArgs := "" + if ysl.hashed { + extraArgs += "--insert-hash" + } + + return c.RunE(ctx, c.Node(c.Spec().NodeCount), fmt.Sprintf( + "./workload init ycsb --insert-count=%d --workload=%s %s {pgurl:1-%d}", + ysl.insertCount, ysl.workload, extraArgs, c.Spec().NodeCount-1)) +} + +func (ysl ycsbSplitLoad) rangeCount(db *gosql.DB) (int, error) { + return rangeCountFrom("ycsb.usertable", db) +} + +func (ysl ycsbSplitLoad) run(ctx context.Context, t test.Test, c cluster.Cluster) error { + extraArgs := "" + if ysl.hashed { + extraArgs += "--insert-hash" + } + + return c.RunE(ctx, c.Node(c.Spec().NodeCount), fmt.Sprintf( + "./workload run ycsb --record-count=%d --workload=%s --concurrency=%d "+ + "--duration='%s' %s {pgurl:1-%d}", + ysl.insertCount, ysl.workload, ysl.concurrency, + ysl.waitDuration.String(), extraArgs, c.Spec().NodeCount-1)) +} + func rangeCountFrom(from string, db *gosql.DB) (int, error) { var ranges int q := fmt.Sprintf("SELECT count(*) FROM [SHOW RANGES FROM TABLE %s]", @@ -85,16 +133,18 @@ func rangeCountFrom(from string, db *gosql.DB) (int, error) { } type splitParams struct { - load splitLoad - maxSize int // The maximum size a range is allowed to be. - qpsThreshold int // QPS Threshold for load based splitting. - cpuThreshold time.Duration // CPU Threshold for load based splitting. - minimumRanges int // Minimum number of ranges expected at the end. - maximumRanges int // Maximum number of ranges expected at the end. + load splitLoad + maxSize int // The maximum size a range is allowed to be. + qpsThreshold int // QPS Threshold for load based splitting. + cpuThreshold time.Duration // CPU Threshold for load based splitting. + minimumRanges int // Minimum number of ranges expected at the end. + maximumRanges int // Maximum number of ranges expected at the end. + initialRangeCount int // Initial range count expected after intiailization. } func registerLoadSplits(r registry.Registry) { - const numNodes = 3 + // Use the 4th node as the workload runner. + const numNodes = 4 r.Add(registry.TestSpec{ Name: fmt.Sprintf("splits/load/uniform/nodes=%d", numNodes), @@ -141,18 +191,37 @@ func registerLoadSplits(r registry.Registry) { }}) }, }) + r.Add(registry.TestSpec{ + Name: fmt.Sprintf("splits/load/uniform/nodes=%d/obj=cpu", numNodes), + Owner: registry.OwnerKV, + Cluster: r.MakeClusterSpec(numNodes), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runLoadSplits(ctx, t, c, splitParams{ + maxSize: 10 << 30, // 10 GB + cpuThreshold: 100 * time.Millisecond, // 1/10th of a CPU per second. + // There should be at least 15 splits, in practice there are on average + // 20. + minimumRanges: 15, + maximumRanges: 25, + load: kvSplitLoad{ + concurrency: 64, // 64 concurrent workers + readPercent: 95, // 95% reads + waitDuration: 10 * time.Minute, + }}) + }, + }) r.Add(registry.TestSpec{ Name: fmt.Sprintf("splits/load/sequential/nodes=%d", numNodes), Owner: registry.OwnerKV, Cluster: r.MakeClusterSpec(numNodes), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runLoadSplits(ctx, t, c, splitParams{ - maxSize: 10 << 30, // 10 GB - qpsThreshold: 100, // 100 queries per second - minimumRanges: 1, // We expect no splits so require only 1 range. + maxSize: 10 << 30, // 10 GB + qpsThreshold: 100, // 100 queries per second // We expect no splits so require only 1 range. However, in practice we // sometimes see a split or two early in, presumably when the sampling // gets lucky. + minimumRanges: 1, maximumRanges: 3, load: kvSplitLoad{ concurrency: 64, // 64 concurrent workers @@ -162,6 +231,30 @@ func registerLoadSplits(r registry.Registry) { }}) }, }) + r.Add(registry.TestSpec{ + Name: fmt.Sprintf("splits/load/sequential/nodes=%d/obj=cpu", numNodes), + Owner: registry.OwnerKV, + Cluster: r.MakeClusterSpec(numNodes), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runLoadSplits(ctx, t, c, splitParams{ + maxSize: 10 << 30, // 10 GB + cpuThreshold: 100 * time.Millisecond, // 1/10th of a CPU per second. + // We expect no splits so require only 1 range, however in practice a + // split may slip in. The reason we don't expect splits for a + // sequential pattern is that existing split samples should only have + // the right counter incremented as new requests come in to the right. + // Any sample we keep should always be the right most request we have + // seen so far. + minimumRanges: 1, + maximumRanges: 5, + load: kvSplitLoad{ + concurrency: 64, + readPercent: 0, + sequential: true, + waitDuration: 60 * time.Second, + }}) + }, + }) r.Add(registry.TestSpec{ Name: fmt.Sprintf("splits/load/spanning/nodes=%d", numNodes), Owner: registry.OwnerKV, @@ -180,6 +273,116 @@ func registerLoadSplits(r registry.Registry) { }}) }, }) + r.Add(registry.TestSpec{ + Name: fmt.Sprintf("splits/load/spanning/nodes=%d/obj=cpu", numNodes), + Owner: registry.OwnerKV, + Cluster: r.MakeClusterSpec(numNodes), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runLoadSplits(ctx, t, c, splitParams{ + maxSize: 10 << 30, // 10 GB + cpuThreshold: 100 * time.Millisecond, // 1/10th of a CPU per second. + // We expect 1-4 splits. There doesn't have the same requirement for + // containment as QPS, instead we want the CPU to be distributed over + // the ranges. i.e. Splitting a range based on QPS when there are only + // scans amplifies the orignal QPS, effectively doubling it. Whereas + // for CPU, the resulting lhs and rhs post split should still add up to + // approx the original range's CPU - when ignoring fixed overhead. + minimumRanges: 2, + maximumRanges: 5, + load: kvSplitLoad{ + concurrency: 64, // 64 concurrent workers + readPercent: 0, // 0% reads + spanPercent: 95, // 95% spanning queries + waitDuration: 60 * time.Second, + }}) + }, + }) + r.Add(registry.TestSpec{ + Name: fmt.Sprintf("splits/load/ycsb/a/nodes=%d/obj=cpu", numNodes), + Owner: registry.OwnerKV, + Cluster: r.MakeClusterSpec(numNodes), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runLoadSplits(ctx, t, c, splitParams{ + maxSize: 10 << 30, // 10 GB + cpuThreshold: 100 * time.Millisecond, // 1/10th of a CPU per second. + // YCSB/A has a zipfian distribution with 50% inserts and 50% updates. + // The number of splits should be between 20-30 after 10 minutes with + // 100ms threshold on 8vCPU machines. + minimumRanges: 20, + maximumRanges: 30, + initialRangeCount: 2, + load: ycsbSplitLoad{ + workload: "a", + concurrency: 64, + insertCount: 1e4, // 100k + waitDuration: 10 * time.Minute, + }}) + }, + }) + r.Add(registry.TestSpec{ + Name: fmt.Sprintf("splits/load/ycsb/b/nodes=%d/obj=cpu", numNodes), + Owner: registry.OwnerKV, + Cluster: r.MakeClusterSpec(numNodes), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runLoadSplits(ctx, t, c, splitParams{ + maxSize: 10 << 30, // 10 GB + // YCSB/B has a zipfian distribution with 95% reads and 5% updates. + // The number of splits should be similar to YCSB/A. + cpuThreshold: 100 * time.Millisecond, // 1/10th of a CPU per second. + minimumRanges: 20, + maximumRanges: 30, + initialRangeCount: 2, + load: ycsbSplitLoad{ + workload: "b", + concurrency: 64, + insertCount: 1e4, // 100k + waitDuration: 10 * time.Minute, + }}) + }, + }) + r.Add(registry.TestSpec{ + Name: fmt.Sprintf("splits/load/ycsb/d/nodes=%d/obj=cpu", numNodes), + Owner: registry.OwnerKV, + Cluster: r.MakeClusterSpec(numNodes), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runLoadSplits(ctx, t, c, splitParams{ + maxSize: 10 << 30, // 10 GB + cpuThreshold: 100 * time.Millisecond, // 1/10th of a CPU per second. + // YCSB/D has a latest distribution i.e. moving hotkey. The inserts are + // hashed - this will lead to many hotspots over the keyspace that + // move. Expect a few less splits than A and B. + minimumRanges: 15, + maximumRanges: 25, + initialRangeCount: 2, + load: ycsbSplitLoad{ + workload: "d", + concurrency: 64, + insertCount: 1e4, // 100k + waitDuration: 10 * time.Minute, + }}) + }, + }) + r.Add(registry.TestSpec{ + Name: fmt.Sprintf("splits/load/ycsb/e/nodes=%d/obj=cpu", numNodes), + Owner: registry.OwnerKV, + Cluster: r.MakeClusterSpec(numNodes), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runLoadSplits(ctx, t, c, splitParams{ + maxSize: 10 << 30, // 10 GB + cpuThreshold: 100 * time.Millisecond, // 1/10th of a CPU per second. + // YCSB/E has a zipfian distribution with 95% scans (limit 1k) and 5% + // inserts. + minimumRanges: 8, + maximumRanges: 15, + initialRangeCount: 2, + load: ycsbSplitLoad{ + workload: "e", + concurrency: 64, + insertCount: 1e4, // 100k + waitDuration: 10 * time.Minute, + }}) + }, + }) } // runLoadSplits tests behavior of load based splitting under @@ -187,8 +390,12 @@ func registerLoadSplits(r registry.Registry) { // splits occur in different workload scenarios. func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params splitParams) { c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(1)) - c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(4)) + startOpts := option.DefaultStartOptsNoBackups() + startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, + "--vmodule=split_queue=2,store_rebalancer=2,allocator=2,replicate_queue=2", + ) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) m := c.NewMonitor(ctx, c.All()) m.Go(func(ctx context.Context) error { @@ -200,9 +407,6 @@ func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params s return err } - // TODO(kvoli): Add load split tests which use CPU, similar to the current - // QPS ones. Tracked by #97540. - // // Set the objective to QPS or CPU and update the load split threshold // appropriately. if params.qpsThreshold > 0 { @@ -237,8 +441,8 @@ func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params s t.Fatalf("failed to set range_max_bytes: %v", err) } } - // Set the range size to a huge size so we don't get splits that occur - // as a result of size thresholds. The kv table will thus be in a single + // Set the range size to a huge size so we don't get splits that occur as a + // result of size thresholds. The workload table will thus be in a single // range unless split by load. setRangeMaxBytes(params.maxSize) @@ -248,8 +452,12 @@ func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params s } t.Status("checking initial range count") - if rc, _ := params.load.rangeCount(db); rc != 1 { - return errors.Errorf("kv.kv table split over multiple ranges.") + expectedInitialRangeCount := params.initialRangeCount + if expectedInitialRangeCount == 0 { + expectedInitialRangeCount = 1 + } + if rc, _ := params.load.rangeCount(db); rc != expectedInitialRangeCount { + return errors.Errorf("table split over multiple ranges (%d)", rc) } t.Status("enable load based splitting")