Skip to content

Commit

Permalink
Merge pull request #16830 from tamird/bench-numbers
Browse files Browse the repository at this point in the history 
*: standardize all sub-benchmarks on key=value format
  • Loading branch information
@tamird
tamird authored Jul 5, 2017
2 parents 644949c + ba6f843 commit 8ebb3ec
Show file tree
Hide file tree
Showing 9 changed files with 309 additions and 195 deletions.
42 changes: 24 additions & 18 deletions pkg/ccl/sqlccl/kv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,30 +105,36 @@ func BenchmarkKV(b *testing.B) {
} {
opName := runtime.FuncForPC(reflect.ValueOf(opFn).Pointer()).Name()
opName = strings.TrimPrefix(opName, "github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.kvInterface.")
for _, rows := range []int{10000} {
b.Run(opName, func(b *testing.B) {
for _, kvFn := range []func(*testing.B) kvInterface{
newKVWriteBatch,
} {
kvTyp := runtime.FuncForPC(reflect.ValueOf(kvFn).Pointer()).Name()
kvTyp = strings.TrimPrefix(kvTyp, "github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.newKV")
for _, initData := range []bool{false} { // TODO(dan): add true
b.Run(fmt.Sprintf("%s%d_%s initData=%t", opName, rows, kvTyp, initData), func(b *testing.B) {
kv := kvFn(b)
defer kv.done()

if err := kv.prep(rows, initData); err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := opFn(kv, rows, i); err != nil {
b.Fatal(err)
b.Run(kvTyp, func(b *testing.B) {
for _, rows := range []int{10000} {
b.Run(fmt.Sprintf("rows=%d", rows), func(b *testing.B) {
for _, initData := range []bool{false, true} {
b.Run(fmt.Sprintf("initData=%t", initData), func(b *testing.B) {
kv := kvFn(b)
defer kv.done()

if err := kv.prep(rows, initData); err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := opFn(kv, rows, i); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
})
}
}
b.StopTimer()
})
}
})
}
})
}
}
})
}
}
75 changes: 38 additions & 37 deletions pkg/ccl/storageccl/bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
package storageccl_test

import (
"fmt"
"path/filepath"
"strconv"
"testing"
Expand All @@ -33,14 +34,14 @@ func BenchmarkAddSSTable(b *testing.B) {
defer dirCleanupFn()

for _, numEntries := range []int{100, 1000, 10000} {
bankData := sampledataccl.BankRows(numEntries)
backupDir := filepath.Join(tempDir, strconv.Itoa(numEntries))
backup, err := sampledataccl.ToBackup(b, bankData, backupDir)
if err != nil {
b.Fatalf("%+v", err)
}

b.Run(strconv.Itoa(numEntries), func(b *testing.B) {
b.Run(fmt.Sprintf("numEntries=%d", numEntries), func(b *testing.B) {
bankData := sampledataccl.BankRows(numEntries)
backupDir := filepath.Join(tempDir, strconv.Itoa(numEntries))
backup, err := sampledataccl.ToBackup(b, bankData, backupDir)
if err != nil {
b.Fatalf("%+v", err)
}

ctx := context.Background()
tc := testcluster.StartTestCluster(b, 3, base.TestClusterArgs{})
defer tc.Stopper().Stop(ctx)
Expand All @@ -49,9 +50,9 @@ func BenchmarkAddSSTable(b *testing.B) {
id := sqlbase.ID(keys.MaxReservedDescID + 1)

var totalLen int64
b.StopTimer()
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
sst, err := engine.MakeRocksDBSstFileWriter()
if err != nil {
b.Fatalf("%+v", err)
Expand All @@ -74,13 +75,13 @@ func BenchmarkAddSSTable(b *testing.B) {
}
sst.Close()
totalLen += int64(len(data))
b.StartTimer()

b.StartTimer()
if err := kvDB.ExperimentalAddSSTable(ctx, span.Key, span.EndKey, data); err != nil {
b.Fatalf("%+v", err)
}
b.StopTimer()
}
b.StopTimer()
b.SetBytes(totalLen / int64(b.N))
})
}
Expand All @@ -91,14 +92,14 @@ func BenchmarkWriteBatch(b *testing.B) {
defer dirCleanupFn()

for _, numEntries := range []int{100, 1000, 10000} {
bankData := sampledataccl.BankRows(numEntries)
backupDir := filepath.Join(tempDir, strconv.Itoa(numEntries))
backup, err := sampledataccl.ToBackup(b, bankData, backupDir)
if err != nil {
b.Fatalf("%+v", err)
}

b.Run(strconv.Itoa(numEntries), func(b *testing.B) {
b.Run(fmt.Sprintf("numEntries=%d", numEntries), func(b *testing.B) {
bankData := sampledataccl.BankRows(numEntries)
backupDir := filepath.Join(tempDir, strconv.Itoa(numEntries))
backup, err := sampledataccl.ToBackup(b, bankData, backupDir)
if err != nil {
b.Fatalf("%+v", err)
}

ctx := context.Background()
tc := testcluster.StartTestCluster(b, 3, base.TestClusterArgs{})
defer tc.Stopper().Stop(ctx)
Expand All @@ -108,9 +109,9 @@ func BenchmarkWriteBatch(b *testing.B) {
var batch engine.RocksDBBatchBuilder

var totalLen int64
b.StopTimer()
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
id++
backup.ResetKeyValueIteration()
kvs, span, err := backup.NextKeyValues(numEntries, id)
Expand All @@ -122,13 +123,13 @@ func BenchmarkWriteBatch(b *testing.B) {
}
repr := batch.Finish()
totalLen += int64(len(repr))
b.StartTimer()

b.StartTimer()
if err := kvDB.WriteBatch(ctx, span.Key, span.EndKey, repr); err != nil {
b.Fatalf("%+v", err)
}
b.StopTimer()
}
b.StopTimer()
b.SetBytes(totalLen / int64(b.N))
})
}
Expand All @@ -150,18 +151,18 @@ func runBenchmarkImport(b *testing.B) {
defer dirCleanupFn()

for _, numEntries := range []int{1, 100, 10000, 100000} {
bankData := sampledataccl.BankRows(numEntries)
backupDir := filepath.Join(tempDir, strconv.Itoa(numEntries))
backup, err := sampledataccl.ToBackup(b, bankData, backupDir)
if err != nil {
b.Fatalf("%+v", err)
}
storage, err := storageccl.ExportStorageConfFromURI(`nodelocal://` + backupDir)
if err != nil {
b.Fatalf("%+v", err)
}

b.Run(strconv.Itoa(numEntries), func(b *testing.B) {
b.Run(fmt.Sprintf("numEntries=%d", numEntries), func(b *testing.B) {
bankData := sampledataccl.BankRows(numEntries)
backupDir := filepath.Join(tempDir, strconv.Itoa(numEntries))
backup, err := sampledataccl.ToBackup(b, bankData, backupDir)
if err != nil {
b.Fatalf("%+v", err)
}
storage, err := storageccl.ExportStorageConfFromURI(`nodelocal://` + backupDir)
if err != nil {
b.Fatalf("%+v", err)
}

ctx := context.Background()
tc := testcluster.StartTestCluster(b, 3, base.TestClusterArgs{})
defer tc.Stopper().Stop(ctx)
Expand All @@ -170,9 +171,9 @@ func runBenchmarkImport(b *testing.B) {
id := sqlbase.ID(keys.MaxReservedDescID + 1)

var totalLen int64
b.StopTimer()
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
id++
var rekeys []roachpb.ImportRequest_TableRekey
var oldStartKey roachpb.Key
Expand All @@ -195,8 +196,8 @@ func runBenchmarkImport(b *testing.B) {
})
}
newStartKey := roachpb.Key(keys.MakeTablePrefix(uint32(id)))
b.StartTimer()

b.StartTimer()
var files []roachpb.ImportRequest_File
for _, file := range backup.Desc.Files {
files = append(files, roachpb.ImportRequest_File{Dir: storage, Path: file.Path})
Expand All @@ -215,8 +216,8 @@ func runBenchmarkImport(b *testing.B) {
b.Fatalf("%+v", pErr.GoError())
}
totalLen += res.(*roachpb.ImportResponse).Imported.DataSize
b.StopTimer()
}
b.StopTimer()
b.SetBytes(totalLen / int64(b.N))
})
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/ccl/storageccl/engineccl/bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func runIterate(

func BenchmarkTimeBoundIterate(b *testing.B) {
for _, loadFactor := range []float32{1.0, 0.5, 0.1, 0.05, 0.0} {
b.Run(fmt.Sprintf("LoadFactor%.2f", loadFactor), func(b *testing.B) {
b.Run(fmt.Sprintf("LoadFactor=%.2f", loadFactor), func(b *testing.B) {
b.Run("NormalIterator", func(b *testing.B) {
runIterate(b, loadFactor, func(e engine.Engine, _, _ hlc.Timestamp) engine.Iterator {
return e.NewIterator(false)
Expand Down
6 changes: 3 additions & 3 deletions pkg/kv/txn_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,12 @@ func TestTxnDBBasics(t *testing.T) {
}
}

// benchmarkSingleRoundtripWithLatency runs a number of transactions writing to
// the same key back to back in a single round-trip. Latency is simulated
// BenchmarkSingleRoundtripWithLatency runs a number of transactions writing
// to the same key back to back in a single round-trip. Latency is simulated
// by pausing before each RPC sent.
func BenchmarkSingleRoundtripWithLatency(b *testing.B) {
for _, latency := range []time.Duration{0, 10 * time.Millisecond} {
b.Run(latency.String(), func(b *testing.B) {
b.Run(fmt.Sprintf("latency=%s", latency), func(b *testing.B) {
var s localtestcluster.LocalTestCluster
s.Latency = latency
s.Start(b, testutils.NewNodeTestBaseContext(), InitSenderForLocalTestCluster)
Expand Down
66 changes: 35 additions & 31 deletions pkg/sql/distsqlrun/sorter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,18 +242,19 @@ func BenchmarkSortAll(b *testing.B) {
post := PostProcessSpec{}

for _, inputSize := range []int{0, 1 << 2, 1 << 4, 1 << 8, 1 << 12, 1 << 16} {
input := make(sqlbase.EncDatumRows, inputSize)
for i := range input {
input[i] = sqlbase.EncDatumRow{
sqlbase.DatumToEncDatum(columnTypeInt, parser.NewDInt(parser.DInt(rng.Int()))),
b.Run(fmt.Sprintf("InputSize=%d", inputSize), func(b *testing.B) {
input := make(sqlbase.EncDatumRows, inputSize)
for i := range input {
input[i] = sqlbase.EncDatumRow{
sqlbase.DatumToEncDatum(columnTypeInt, parser.NewDInt(parser.DInt(rng.Int()))),
}
}
}
rowSource := NewRepeatableRowSource(types, input)
s, err := newSorter(&flowCtx, &spec, rowSource, &post, &RowDisposer{})
if err != nil {
b.Fatal(err)
}
b.Run(fmt.Sprintf("InputSize%d", inputSize), func(b *testing.B) {
rowSource := NewRepeatableRowSource(types, input)
s, err := newSorter(&flowCtx, &spec, rowSource, &post, &RowDisposer{})
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.Run(ctx, nil)
rowSource.Reset()
Expand Down Expand Up @@ -281,27 +282,30 @@ func BenchmarkSortLimit(b *testing.B) {
OutputOrdering: convertToSpecOrdering(sqlbase.ColumnOrdering{{ColIdx: 0, Direction: encoding.Ascending}}),
}

inputSize := 1 << 16
input := make(sqlbase.EncDatumRows, inputSize)
for i := range input {
input[i] = sqlbase.EncDatumRow{
sqlbase.DatumToEncDatum(columnTypeInt, parser.NewDInt(parser.DInt(rng.Int()))),
const inputSize = 1 << 16
b.Run(fmt.Sprintf("InputSize=%d", inputSize), func(b *testing.B) {
input := make(sqlbase.EncDatumRows, inputSize)
for i := range input {
input[i] = sqlbase.EncDatumRow{
sqlbase.DatumToEncDatum(columnTypeInt, parser.NewDInt(parser.DInt(rng.Int()))),
}
}
}
rowSource := NewRepeatableRowSource(types, input)
rowSource := NewRepeatableRowSource(types, input)

for _, limit := range []uint64{1, 1 << 2, 1 << 4, 1 << 8, 1 << 12, 1 << 16} {
s, err := newSorter(
&flowCtx, &spec, rowSource, &PostProcessSpec{Limit: limit}, &RowDisposer{},
)
if err != nil {
b.Fatal(err)
for _, limit := range []uint64{1, 1 << 2, 1 << 4, 1 << 8, 1 << 12, 1 << 16} {
b.Run(fmt.Sprintf("Limit=%d", limit), func(b *testing.B) {
s, err := newSorter(
&flowCtx, &spec, rowSource, &PostProcessSpec{Limit: limit}, &RowDisposer{},
)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.Run(ctx, nil)
rowSource.Reset()
}
})
}
b.Run(fmt.Sprintf("InputSize%dLimit%d", inputSize, limit), func(b *testing.B) {
for i := 0; i < b.N; i++ {
s.Run(ctx, nil)
rowSource.Reset()
}
})
}
})
}
32 changes: 18 additions & 14 deletions pkg/sql/kv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,29 +280,33 @@ func BenchmarkKV(b *testing.B) {
} {
opName := runtime.FuncForPC(reflect.ValueOf(opFn).Pointer()).Name()
opName = strings.TrimPrefix(opName, "github.com/cockroachdb/cockroach/pkg/sql_test.kvInterface.")
for _, rows := range []int{1, 10, 100, 1000, 10000} {
b.Run(opName, func(b *testing.B) {
for _, kvFn := range []func(*testing.B) kvInterface{
newKVNative,
newKVSQL,
} {
kvTyp := runtime.FuncForPC(reflect.ValueOf(kvFn).Pointer()).Name()
kvTyp = strings.TrimPrefix(kvTyp, "github.com/cockroachdb/cockroach/pkg/sql_test.newKV")
b.Run(fmt.Sprintf("%s%d_%s", opName, rows, kvTyp), func(b *testing.B) {
kv := kvFn(b)
defer kv.done()
b.Run(kvTyp, func(b *testing.B) {
for _, rows := range []int{1, 10, 100, 1000, 10000} {
b.Run(fmt.Sprintf("rows=%d", rows), func(b *testing.B) {
kv := kvFn(b)
defer kv.done()

if err := kv.prep(rows, i != 0 /* Insert */ && i != 2 /* Delete */); err != nil {
b.Fatal(err)
if err := kv.prep(rows, i != 0 /* Insert */ && i != 2 /* Delete */); err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := opFn(kv, rows, i); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
})
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := opFn(kv, rows, i); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
})
}
}
})
}
}
Loading

0 comments on commit 8ebb3ec

Please sign in to comment.