Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve commit log reader speed #1724

Merged
merged 6 commits into from
Jun 13, 2019
+417 −376
Merged

Improve commit log reader speed #1724

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
04e4e76
Refactor reader to use decode fast code paths and reduce to single th…
robskillington Jun 4, 2019
55579d5
fix bug in how seriesPredicate was being called
Jun 11, 2019
b4c80a3
Feedback
robskillington Jun 12, 2019
2aa9af1
Add bootstrap perf integration test
robskillington Jun 12, 2019
cec3f06
Use fixed size heap to generate the data
robskillington Jun 12, 2019
d499f64
Merge branch 'master' into r/commit-log-bootstrapper-enhancements
robskillington Jun 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions src/dbnode/persist/fs/commitlog/commit_log_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,22 @@ import (
"github.com/uber-go/tally"
)

// readAllSeriesPredicateTest is the same as ReadAllSeriesPredicate except
// it asserts that the ID and the namespace are not nil.
func readAllSeriesPredicateTest() SeriesFilterPredicate {
return func(id ident.ID, namespace ident.ID) bool {
if id == nil {
panic(fmt.Sprintf("series ID passed to series predicate is nil"))
}

if namespace == nil {
panic(fmt.Sprintf("namespace ID passed to series predicate is nil"))
}

return true
}
}

type overrides struct {
clock *mclock.Mock
flushInterval *time.Duration
Expand Down Expand Up @@ -286,7 +302,7 @@ func assertCommitLogWritesByIterating(t *testing.T, l *commitLog, writes []testW
iterOpts := IteratorOpts{
CommitLogOptions: l.opts,
FileFilterPredicate: ReadAllPredicate(),
SeriesFilterPredicate: ReadAllSeriesPredicate(),
SeriesFilterPredicate: readAllSeriesPredicateTest(),
}
iter, corruptFiles, err := NewIterator(iterOpts)
require.NoError(t, err)
Expand Down Expand Up @@ -413,7 +429,7 @@ func TestReadCommitLogMissingMetadata(t *testing.T) {
iterOpts := IteratorOpts{
CommitLogOptions: opts,
FileFilterPredicate: ReadAllPredicate(),
SeriesFilterPredicate: ReadAllSeriesPredicate(),
SeriesFilterPredicate: readAllSeriesPredicateTest(),
}
iter, corruptFiles, err := NewIterator(iterOpts)
require.NoError(t, err)
Expand Down Expand Up @@ -461,7 +477,7 @@ func TestCommitLogReaderIsNotReusable(t *testing.T) {
require.Equal(t, 2, len(files))

// Assert commitlog cannot be opened more than once
reader := newCommitLogReader(opts, ReadAllSeriesPredicate())
reader := newCommitLogReader(opts, readAllSeriesPredicateTest())
_, err = reader.Open(files[0])
require.NoError(t, err)
reader.Close()
Expand Down Expand Up @@ -519,7 +535,7 @@ func TestCommitLogIteratorUsesPredicateFilterForNonCorruptFiles(t *testing.T) {
iterOpts := IteratorOpts{
CommitLogOptions: opts,
FileFilterPredicate: commitLogPredicate,
SeriesFilterPredicate: ReadAllSeriesPredicate(),
SeriesFilterPredicate: readAllSeriesPredicateTest(),
}
iter, corruptFiles, err := NewIterator(iterOpts)
require.NoError(t, err)
Expand Down Expand Up @@ -563,7 +579,7 @@ func TestCommitLogIteratorUsesPredicateFilterForCorruptFiles(t *testing.T) {
iterOpts := IteratorOpts{
CommitLogOptions: opts,
FileFilterPredicate: ReadAllPredicate(),
SeriesFilterPredicate: ReadAllSeriesPredicate(),
SeriesFilterPredicate: readAllSeriesPredicateTest(),
}
iter, corruptFiles, err := NewIterator(iterOpts)
require.NoError(t, err)
Expand All @@ -580,7 +596,7 @@ func TestCommitLogIteratorUsesPredicateFilterForCorruptFiles(t *testing.T) {
iterOpts = IteratorOpts{
CommitLogOptions: opts,
FileFilterPredicate: ignoreCorruptPredicate,
SeriesFilterPredicate: ReadAllSeriesPredicate(),
SeriesFilterPredicate: readAllSeriesPredicateTest(),
}
iter, corruptFiles, err = NewIterator(iterOpts)
require.NoError(t, err)
Expand Down
Loading