[dbnode] Load and cache info files during bootstrap (#2598)

src/cmd/services/m3dbnode/config/bootstrap.go

@@ -283,7 +283,7 @@ func (bsc BootstrapConfiguration) New(
 	if bsc.CacheSeriesMetadata != nil {
 		providerOpts = providerOpts.SetCacheSeriesMetadata(*bsc.CacheSeriesMetadata)
 	}
-	return bootstrap.NewProcessProvider(bs, providerOpts, rsOpts)
+	return bootstrap.NewProcessProvider(bs, providerOpts, rsOpts, fsOpts)
 }
 
 func (bsc BootstrapConfiguration) filesystemConfig() BootstrapFilesystemConfiguration {

src/dbnode/integration/bootstrap_after_buffer_rotation_regression_test.go

@@ -125,6 +125,7 @@ func TestBootstrapAfterBufferRotation(t *testing.T) {
 		read: func(
 			ctx context.Context,
 			namespaces bootstrap.Namespaces,
+			cache bootstrap.Cache,
 		) (bootstrap.NamespaceResults, error) {
 			<-signalCh
 			// Mark all as unfulfilled so the commitlog bootstrapper will be called after
@@ -133,7 +134,7 @@ func TestBootstrapAfterBufferRotation(t *testing.T) {
 			if err != nil {
 				return bootstrap.NamespaceResults{}, err
 			}
-			return bs.Bootstrap(ctx, namespaces)
+			return bs.Bootstrap(ctx, namespaces, cache)
 		},
 	}, bootstrapOpts, bs)
 
@@ -142,7 +143,7 @@ func TestBootstrapAfterBufferRotation(t *testing.T) {
 		SetOrigin(setup.Origin())
 
 	processProvider, err := bootstrap.NewProcessProvider(
-		test, processOpts, bootstrapOpts)
+		test, processOpts, bootstrapOpts, fsOpts)
 	require.NoError(t, err)
 	setup.SetStorageOpts(setup.StorageOpts().SetBootstrapProcessProvider(processProvider))
 

src/dbnode/integration/bootstrap_before_buffer_rotation_no_tick_regression_test.go

@@ -139,6 +139,7 @@ func TestBootstrapBeforeBufferRotationNoTick(t *testing.T) {
 		read: func(
 			ctx context.Context,
 			namespaces bootstrap.Namespaces,
+			cache bootstrap.Cache,
 		) (bootstrap.NamespaceResults, error) {
 			<-signalCh
 			// Mark all as unfulfilled so the commitlog bootstrapper will be called after
@@ -147,14 +148,14 @@ func TestBootstrapBeforeBufferRotationNoTick(t *testing.T) {
 			if err != nil {
 				return bootstrap.NamespaceResults{}, err
 			}
-			return bs.Bootstrap(ctx, namespaces)
+			return bs.Bootstrap(ctx, namespaces, cache)
 		},
 	}, bootstrapOpts, bs)
 
 	processOpts := bootstrap.NewProcessOptions().
 		SetTopologyMapProvider(setup).
 		SetOrigin(setup.Origin())
-	process, err := bootstrap.NewProcessProvider(test, processOpts, bootstrapOpts)
+	process, err := bootstrap.NewProcessProvider(test, processOpts, bootstrapOpts, fsOpts)
 	require.NoError(t, err)
 	setup.SetStorageOpts(setup.StorageOpts().SetBootstrapProcessProvider(process))
 

src/dbnode/integration/bootstrap_helpers.go

@@ -45,23 +45,23 @@ func newTestBootstrapperSource(
 	if opts.availableData != nil {
 		src.availableData = opts.availableData
 	} else {
-		src.availableData = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
+		src.availableData = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.Cache, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
 			return shardsTimeRanges, nil
 		}
 	}
 
 	if opts.availableIndex != nil {
 		src.availableIndex = opts.availableIndex
 	} else {
-		src.availableIndex = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
+		src.availableIndex = func(_ namespace.Metadata, shardsTimeRanges result.ShardTimeRanges, _ bootstrap.Cache, _ bootstrap.RunOptions) (result.ShardTimeRanges, error) {
 			return shardsTimeRanges, nil
 		}
 	}
 
 	if opts.read != nil {
 		src.read = opts.read
 	} else {
-		src.read = func(ctx context.Context, namespaces bootstrap.Namespaces) (bootstrap.NamespaceResults, error) {
+		src.read = func(_ context.Context, namespaces bootstrap.Namespaces, _ bootstrap.Cache) (bootstrap.NamespaceResults, error) {
 			return bootstrap.NewNamespaceResults(namespaces), nil
 		}
 	}
@@ -96,40 +96,43 @@ type testBootstrapper struct {
 }
 
 type testBootstrapperSourceOptions struct {
-	availableData  func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
-	availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
-	read           func(ctx context.Context, namespaces bootstrap.Namespaces) (bootstrap.NamespaceResults, error)
+	availableData  func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
+	availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
+	read           func(ctx context.Context, namespaces bootstrap.Namespaces, cache bootstrap.Cache) (bootstrap.NamespaceResults, error)
 }
 
 var _ bootstrap.Source = &testBootstrapperSource{}
 
 type testBootstrapperSource struct {
-	availableData  func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
-	availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.RunOptions) (result.ShardTimeRanges, error)
-	read           func(ctx context.Context, namespaces bootstrap.Namespaces) (bootstrap.NamespaceResults, error)
+	availableData  func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
+	availableIndex func(namespace.Metadata, result.ShardTimeRanges, bootstrap.Cache, bootstrap.RunOptions) (result.ShardTimeRanges, error)
+	read           func(ctx context.Context, namespaces bootstrap.Namespaces, cache bootstrap.Cache) (bootstrap.NamespaceResults, error)
 }
 
 func (t testBootstrapperSource) AvailableData(
 	ns namespace.Metadata,
 	shardsTimeRanges result.ShardTimeRanges,
+	cache bootstrap.Cache,
 	runOpts bootstrap.RunOptions,
 ) (result.ShardTimeRanges, error) {
-	return t.availableData(ns, shardsTimeRanges, runOpts)
+	return t.availableData(ns, shardsTimeRanges, cache, runOpts)
 }
 
 func (t testBootstrapperSource) AvailableIndex(
 	ns namespace.Metadata,
 	shardsTimeRanges result.ShardTimeRanges,
+	cache bootstrap.Cache,
 	runOpts bootstrap.RunOptions,
 ) (result.ShardTimeRanges, error) {
-	return t.availableIndex(ns, shardsTimeRanges, runOpts)
+	return t.availableIndex(ns, shardsTimeRanges, cache, runOpts)
 }
 
 func (t testBootstrapperSource) Read(
 	ctx context.Context,
 	namespaces bootstrap.Namespaces,
+	cache bootstrap.Cache,
 ) (bootstrap.NamespaceResults, error) {
-	return t.read(ctx, namespaces)
+	return t.read(ctx, namespaces, cache)
 }
 
 func (t testBootstrapperSource) String() string {

src/dbnode/integration/integration.go

@@ -299,7 +299,7 @@ func newDefaultBootstrappableTestSetups(
 		processOpts := bootstrap.NewProcessOptions().
 			SetTopologyMapProvider(setup).
 			SetOrigin(setup.Origin())
-		provider, err := bootstrap.NewProcessProvider(fsBootstrapper, processOpts, bsOpts)
+		provider, err := bootstrap.NewProcessProvider(fsBootstrapper, processOpts, bsOpts, fsOpts)
 		require.NoError(t, err)
 
 		setup.SetStorageOpts(setup.StorageOpts().SetBootstrapProcessProvider(provider))

src/dbnode/integration/setup.go

@@ -952,7 +952,7 @@ func (ts *testSetup) InitializeBootstrappers(opts InitializeBootstrappersOptions
 	processOpts := bootstrap.NewProcessOptions().
 		SetTopologyMapProvider(ts).
 		SetOrigin(ts.Origin())
-	process, err := bootstrap.NewProcessProvider(bs, processOpts, bsOpts)
+	process, err := bootstrap.NewProcessProvider(bs, processOpts, bsOpts, fsOpts)
 	if err != nil {
 		return err
 	}

src/dbnode/persist/fs/types.go

@@ -679,9 +679,3 @@ type CrossBlockIterator interface {
 	// Reset resets the iterator to the given block records.
 	Reset(records []BlockRecord)
 }
-
-// InfoFileResultsPerShard maps shards to info files.
-type InfoFileResultsPerShard map[uint32][]ReadInfoFileResult
-
-// InfoFilesByNamespace maps a namespace to info files grouped by shard.
-type InfoFilesByNamespace map[namespace.Metadata]InfoFileResultsPerShard