Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[dbnode] Load and cache info files during bootstrap #2598

Merged
merged 6 commits into from
Sep 24, 2020
Merged

[dbnode] Load and cache info files during bootstrap #2598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
07b6e73
Add bootstrap.State and thread through bootstrap process
nbroyles Sep 15, 2020
017bf79
Fix tests due to addition of bootstrap.State to bootstrap.Source inte…
nbroyles Sep 15, 2020
724b75e
Rename boostrap.State to bootstrap.Cache
nbroyles Sep 23, 2020
e329e9e
Return err from Cache methods
nbroyles Sep 23, 2020
c8fda13
Rename InfoFilesFinder to NamespaceDetails
nbroyles Sep 23, 2020
cbb89aa
Fix remaining references to state instead of cache
nbroyles Sep 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/cmd/services/m3dbnode/config/bootstrap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ func (bsc BootstrapConfiguration) New(
if bsc.CacheSeriesMetadata != nil {
providerOpts = providerOpts.SetCacheSeriesMetadata(*bsc.CacheSeriesMetadata)
}
return bootstrap.NewProcessProvider(bs, providerOpts, rsOpts)
return bootstrap.NewProcessProvider(bs, providerOpts, rsOpts, fsOpts)
}

func (bsc BootstrapConfiguration) filesystemConfig() BootstrapFilesystemConfiguration {
Expand Down
5 changes: 3 additions & 2 deletions src/dbnode/integration/bootstrap_after_buffer_rotation_regression_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ func TestBootstrapAfterBufferRotation(t *testing.T) {
read: func(
ctx context.Context,
namespaces bootstrap.Namespaces,
cache bootstrap.Cache,
) (bootstrap.NamespaceResults, error) {
<-signalCh
// Mark all as unfulfilled so the commitlog bootstrapper will be called after
Expand All @@ -133,7 +134,7 @@ func TestBootstrapAfterBufferRotation(t *testing.T) {
if err != nil {
return bootstrap.NamespaceResults{}, err
}
return bs.Bootstrap(ctx, namespaces)
return bs.Bootstrap(ctx, namespaces, cache)
},
}, bootstrapOpts, bs)

Expand All @@ -142,7 +143,7 @@ func TestBootstrapAfterBufferRotation(t *testing.T) {
SetOrigin(setup.Origin())

processProvider, err := bootstrap.NewProcessProvider(
test, processOpts, bootstrapOpts)
test, processOpts, bootstrapOpts, fsOpts)
require.NoError(t, err)
setup.SetStorageOpts(setup.StorageOpts().SetBootstrapProcessProvider(processProvider))

Expand Down
5 changes: 3 additions & 2 deletions src/dbnode/integration/bootstrap_before_buffer_rotation_no_tick_regression_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ func TestBootstrapBeforeBufferRotationNoTick(t *testing.T) {
read: func(
ctx context.Context,
namespaces bootstrap.Namespaces,
cache bootstrap.Cache,
) (bootstrap.NamespaceResults, error) {
<-signalCh
// Mark all as unfulfilled so the commitlog bootstrapper will be called after
Expand All @@ -147,14 +148,14 @@ func TestBootstrapBeforeBufferRotationNoTick(t *testing.T) {
if err != nil {
return bootstrap.NamespaceResults{}, err
}
return bs.Bootstrap(ctx, namespaces)
return bs.Bootstrap(ctx, namespaces, cache)
},
}, bootstrapOpts, bs)

processOpts := bootstrap.NewProcessOptions().
SetTopologyMapProvider(setup).
SetOrigin(setup.Origin())
process, err := bootstrap.NewProcessProvider(test, processOpts, bootstrapOpts)
process, err := bootstrap.NewProcessProvider(test, processOpts, bootstrapOpts, fsOpts)
require.NoError(t, err)
setup.SetStorageOpts(setup.StorageOpts().SetBootstrapProcessProvider(process))

Expand Down
27 changes: 15 additions & 12 deletions src/dbnode/integration/bootstrap_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,23 +45,23 @@ func newTestBootstrapperSource(
if opts.availableData != nil {
src.availableData = opts.availableData
} else {
src.availableData = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
src.availableData = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.Cache, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
return shardsTimeRanges, nil
}
}

if opts.availableIndex != nil {
src.availableIndex = opts.availableIndex
} else {
src.availableIndex = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
src.availableIndex = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.Cache, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
return shardsTimeRanges, nil
}
}

if opts.read != nil {
src.read = opts.read
} else {
src.read = func(ctx context.Context, namespaces bootstrap.Namespaces) (bootstrap.NamespaceResults, error) {
src.read = func(_ context.Context, namespaces bootstrap.Namespaces, _ bootstrap.Cache) (bootstrap.NamespaceResults, error) {
return bootstrap.NewNamespaceResults(namespaces), nil
}
}
Expand Down Expand Up @@ -96,40 +96,43 @@ type testBootstrapper struct {
}

type testBootstrapperSourceOptions struct {
availableData func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
read func(ctx context.Context, namespaces bootstrap.Namespaces) (bootstrap.NamespaceResults, error)
availableData func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
read func(ctx context.Context, namespaces bootstrap.Namespaces, cache bootstrap.Cache) (bootstrap.NamespaceResults, error)
}

var _ bootstrap.Source = &testBootstrapperSource{}

type testBootstrapperSource struct {
availableData func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
read func(ctx context.Context, namespaces bootstrap.Namespaces) (bootstrap.NamespaceResults, error)
availableData func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
read func(ctx context.Context, namespaces bootstrap.Namespaces, cache bootstrap.Cache) (bootstrap.NamespaceResults, error)
}

func (t testBootstrapperSource) AvailableData(
ns namespace.Metadata,
shardsTimeRanges result.ShardTimeRanges,
cache bootstrap.Cache,
runOpts bootstrap.RunOptions,
) (result.ShardTimeRanges, error) {
return t.availableData(ns, shardsTimeRanges, runOpts)
return t.availableData(ns, shardsTimeRanges, cache, runOpts)
}

func (t testBootstrapperSource) AvailableIndex(
ns namespace.Metadata,
shardsTimeRanges result.ShardTimeRanges,
cache bootstrap.Cache,
runOpts bootstrap.RunOptions,
) (result.ShardTimeRanges, error) {
return t.availableIndex(ns, shardsTimeRanges, runOpts)
return t.availableIndex(ns, shardsTimeRanges, cache, runOpts)
}

func (t testBootstrapperSource) Read(
ctx context.Context,
namespaces bootstrap.Namespaces,
cache bootstrap.Cache,
) (bootstrap.NamespaceResults, error) {
return t.read(ctx, namespaces)
return t.read(ctx, namespaces, cache)
}

func (t testBootstrapperSource) String() string {
Expand Down
2 changes: 1 addition & 1 deletion src/dbnode/integration/integration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ func newDefaultBootstrappableTestSetups(
processOpts := bootstrap.NewProcessOptions().
SetTopologyMapProvider(setup).
SetOrigin(setup.Origin())
provider, err := bootstrap.NewProcessProvider(fsBootstrapper, processOpts, bsOpts)
provider, err := bootstrap.NewProcessProvider(fsBootstrapper, processOpts, bsOpts, fsOpts)
require.NoError(t, err)

setup.SetStorageOpts(setup.StorageOpts().SetBootstrapProcessProvider(provider))
Expand Down
2 changes: 1 addition & 1 deletion src/dbnode/integration/setup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -952,7 +952,7 @@ func (ts *testSetup) InitializeBootstrappers(opts InitializeBootstrappersOptions
processOpts := bootstrap.NewProcessOptions().
SetTopologyMapProvider(ts).
SetOrigin(ts.Origin())
process, err := bootstrap.NewProcessProvider(bs, processOpts, bsOpts)
process, err := bootstrap.NewProcessProvider(bs, processOpts, bsOpts, fsOpts)
if err != nil {
return err
}
Expand Down
6 changes: 0 additions & 6 deletions src/dbnode/persist/fs/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -679,9 +679,3 @@ type CrossBlockIterator interface {
// Reset resets the iterator to the given block records.
Reset(records []BlockRecord)
}

// InfoFileResultsPerShard maps shards to info files.
type InfoFileResultsPerShard map[uint32][]ReadInfoFileResult

// InfoFilesByNamespace maps a namespace to info files grouped by shard.
type InfoFilesByNamespace map[namespace.Metadata]InfoFileResultsPerShard
155 changes: 139 additions & 16 deletions src/dbnode/storage/bootstrap/bootstrap_mock.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading