forked from cockroachdb/pebble
-
Notifications
You must be signed in to change notification settings - Fork 0
/
options_test.go
324 lines (294 loc) · 8.03 KB
/
options_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"math/rand"
"runtime"
"testing"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/vfs"
"github.com/stretchr/testify/require"
)
// testingRandomized randomizes some default options. Currently, it's
// used for testing under a random format major version in some tests.
func (o *Options) testingRandomized(t testing.TB) *Options {
if o == nil {
o = &Options{}
}
if o.Logger == nil {
o.Logger = testLogger{t: t}
}
if o.FormatMajorVersion == FormatDefault {
// Pick a random format major version from the range
// [FormatMinSupported, FormatNewest].
n := rand.Intn(int(internalFormatNewest - FormatMinSupported + 1))
o.FormatMajorVersion = FormatMinSupported + FormatMajorVersion(n)
t.Logf("Running %s with format major version %s", t.Name(), o.FormatMajorVersion.String())
}
return o
}
func testingRandomized(t testing.TB, o *Options) *Options {
o.testingRandomized(t)
return o
}
func TestLevelOptions(t *testing.T) {
var opts *Options
opts = opts.EnsureDefaults()
testCases := []struct {
level int
targetFileSize int64
}{
{0, 2 << 20},
{1, (2 * 2) << 20},
{2, (4 * 2) << 20},
{3, (8 * 2) << 20},
{4, (16 * 2) << 20},
{5, (32 * 2) << 20},
{6, (64 * 2) << 20},
}
for _, c := range testCases {
l := opts.Level(c.level)
if c.targetFileSize != l.TargetFileSize {
t.Fatalf("%d: expected target-file-size %d, but found %d",
c.level, c.targetFileSize, l.TargetFileSize)
}
}
}
func TestOptionsString(t *testing.T) {
n := runtime.GOMAXPROCS(8)
defer runtime.GOMAXPROCS(n)
const expected = `[Version]
pebble_version=0.1
[Options]
bytes_per_sync=524288
cache_size=8388608
cleaner=delete
compaction_debt_concurrency=1073741824
comparer=leveldb.BytewiseComparator
disable_wal=false
flush_delay_delete_range=0s
flush_delay_range_key=0s
flush_split_bytes=4194304
format_major_version=13
l0_compaction_concurrency=10
l0_compaction_file_threshold=500
l0_compaction_threshold=4
l0_stop_writes_threshold=12
lbase_max_bytes=67108864
max_concurrent_compactions=1
max_manifest_file_size=134217728
max_open_files=1000
mem_table_size=4194304
mem_table_stop_writes_threshold=2
min_deletion_rate=0
merger=pebble.concatenate
multilevel_compaction_heuristic=wamp(0.00, false)
read_compaction_rate=16000
read_sampling_multiplier=16
strict_wal_tail=true
table_cache_shards=8
validate_on_ingest=false
wal_dir=
wal_bytes_per_sync=0
max_writer_concurrency=0
force_writer_parallelism=false
secondary_cache_size_bytes=0
create_on_shared=0
[Level "0"]
block_restart_interval=16
block_size=4096
block_size_threshold=90
compression=Snappy
filter_policy=none
filter_type=table
index_block_size=4096
target_file_size=2097152
`
var opts *Options
opts = opts.EnsureDefaults()
require.Equal(t, expected, opts.String())
}
func TestOptionsCheck(t *testing.T) {
var opts *Options
opts = opts.EnsureDefaults()
s := opts.String()
require.NoError(t, opts.Check(s))
require.Regexp(t, `invalid key=value syntax`, opts.Check("foo\n"))
tmp := *opts
tmp.Comparer = &Comparer{Name: "foo"}
require.Regexp(t, `comparer name from file.*!=.*`, tmp.Check(s))
tmp = *opts
tmp.Merger = &Merger{Name: "foo"}
require.Regexp(t, `merger name from file.*!=.*`, tmp.Check(s))
// RocksDB uses a similar (INI-style) syntax for the OPTIONS file, but
// different section names and keys.
s = `
[CFOptions "default"]
comparator=rocksdb-comparer
merge_operator=rocksdb-merger
`
tmp = *opts
tmp.Comparer = &Comparer{Name: "foo"}
require.Regexp(t, `comparer name from file.*!=.*`, tmp.Check(s))
tmp.Comparer = &Comparer{Name: "rocksdb-comparer"}
tmp.Merger = &Merger{Name: "foo"}
require.Regexp(t, `merger name from file.*!=.*`, tmp.Check(s))
tmp.Merger = &Merger{Name: "rocksdb-merger"}
require.NoError(t, tmp.Check(s))
// RocksDB allows the merge operator to be unspecified, in which case it
// shows up as "nullptr".
s = `
[CFOptions "default"]
merge_operator=nullptr
`
tmp = *opts
require.NoError(t, tmp.Check(s))
}
type testCleaner struct{}
func (testCleaner) Clean(fs vfs.FS, fileType base.FileType, path string) error {
return nil
}
func (testCleaner) String() string {
return "test-cleaner"
}
func TestOptionsParse(t *testing.T) {
testComparer := *DefaultComparer
testComparer.Name = "test-comparer"
testMerger := *DefaultMerger
testMerger.Name = "test-merger"
var newCacheSize int64
hooks := &ParseHooks{
NewCache: func(size int64) *Cache {
newCacheSize = size
return nil
},
NewCleaner: func(name string) (Cleaner, error) {
if name == (testCleaner{}).String() {
return testCleaner{}, nil
}
return nil, errors.Errorf("unknown cleaner: %q", name)
},
NewComparer: func(name string) (*Comparer, error) {
if name == testComparer.Name {
return &testComparer, nil
}
return nil, errors.Errorf("unknown comparer: %q", name)
},
NewMerger: func(name string) (*Merger, error) {
if name == testMerger.Name {
return &testMerger, nil
}
return nil, errors.Errorf("unknown merger: %q", name)
},
}
testCases := []struct {
cleaner Cleaner
comparer *Comparer
merger *Merger
}{
{testCleaner{}, nil, nil},
{nil, &testComparer, nil},
{nil, nil, &testMerger},
}
for _, c := range testCases {
t.Run("", func(t *testing.T) {
var opts Options
opts.Comparer = c.comparer
opts.Merger = c.merger
opts.WALDir = "wal"
opts.Levels = make([]LevelOptions, 3)
opts.Levels[0].BlockSize = 1024
opts.Levels[1].BlockSize = 2048
opts.Levels[2].BlockSize = 4096
opts.Experimental.CompactionDebtConcurrency = 100
opts.FlushDelayDeleteRange = 10 * time.Second
opts.FlushDelayRangeKey = 11 * time.Second
opts.Experimental.LevelMultiplier = 5
opts.TargetByteDeletionRate = 200
opts.Experimental.ReadCompactionRate = 300
opts.Experimental.ReadSamplingMultiplier = 400
opts.Experimental.TableCacheShards = 500
opts.Experimental.MaxWriterConcurrency = 1
opts.Experimental.ForceWriterParallelism = true
opts.Experimental.SecondaryCacheSizeBytes = 1024
opts.EnsureDefaults()
str := opts.String()
newCacheSize = 0
var parsedOptions Options
require.NoError(t, parsedOptions.Parse(str, hooks))
parsedStr := parsedOptions.String()
if str != parsedStr {
t.Fatalf("expected\n%s\nbut found\n%s", str, parsedStr)
}
require.Nil(t, parsedOptions.Cache)
require.NotEqual(t, newCacheSize, 0)
})
}
}
func TestOptionsValidate(t *testing.T) {
testCases := []struct {
options string
expected string
}{
{``, ``},
{`
[Options]
l0_compaction_concurrency=0
`,
`L0CompactionConcurrency \(0\) must be >= 1`,
},
{`
[Options]
l0_compaction_threshold=2
l0_stop_writes_threshold=1
`,
`L0StopWritesThreshold .* must be >= L0CompactionThreshold .*`,
},
{`
[Options]
mem_table_size=4294967296
`,
`MemTableSize \(4\.0GB\) must be < [2|4]\.0GB`,
},
{`
[Options]
mem_table_stop_writes_threshold=1
`,
`MemTableStopWritesThreshold .* must be >= 2`,
},
}
for _, c := range testCases {
t.Run("", func(t *testing.T) {
var opts Options
opts.EnsureDefaults()
require.NoError(t, opts.Parse(c.options, nil))
err := opts.Validate()
if c.expected == "" {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Regexp(t, c.expected, err.Error())
}
})
}
}
// This test isn't being done in TestOptionsValidate
// cause it doesn't support setting pointers.
func TestOptionsValidateCache(t *testing.T) {
var opts Options
opts.EnsureDefaults()
opts.Cache = NewCache(8 << 20)
defer opts.Cache.Unref()
opts.TableCache = NewTableCache(NewCache(8<<20), 10, 1)
defer opts.TableCache.cache.Unref()
defer opts.TableCache.Unref()
err := opts.Validate()
require.Error(t, err)
if fmt.Sprint(err) != "underlying cache in the TableCache and the Cache dont match" {
t.Errorf("Unexpected error message")
}
}