[dbnode] Use active index block which GCs expired series instead of e…

…xplicit block rotations (#3464)

site/content/overview/components.md

@@ -17,4 +17,4 @@ M3DB is a distributed time series database that provides scalable storage and a
 
 ## M3 Aggregator
 
-{{< fileinclude file="m3aggregator_intro.md" >}}
+{{< fileinclude file="m3aggregator_intro.md" >}}

src/dbnode/generated/mocks/generate.go

@@ -24,7 +24,7 @@
 //go:generate sh -c "mockgen -package=xio $PACKAGE/src/dbnode/x/xio SegmentReader,SegmentReaderPool | genclean -pkg $PACKAGE/src/dbnode/x/xio -out $GOPATH/src/$PACKAGE/src/dbnode/x/xio/io_mock.go"
 //go:generate sh -c "mockgen -package=digest -destination=$GOPATH/src/$PACKAGE/src/dbnode/digest/digest_mock.go $PACKAGE/src/dbnode/digest ReaderWithDigest"
 //go:generate sh -c "mockgen -package=series $PACKAGE/src/dbnode/storage/series DatabaseSeries,QueryableBlockRetriever | genclean -pkg $PACKAGE/src/dbnode/storage/series -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/series_mock.go"
-//go:generate sh -c "mockgen -package=lookup $PACKAGE/src/dbnode/storage/series/lookup IndexWriter | genclean -pkg $PACKAGE/src/dbnode/storage/series/lookup -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/lookup/lookup_mock.go"
+//go:generate sh -c "mockgen -package=storage $PACKAGE/src/dbnode/storage IndexWriter | genclean -pkg $PACKAGE/src/dbnode/storage -out $GOPATH/src/$PACKAGE/src/dbnode/storage/lookup_mock.go"
 
 // mockgen rules for generating mocks for unexported interfaces (file mode)
 //go:generate sh -c "mockgen -package=encoding -destination=$GOPATH/src/$PACKAGE/src/dbnode/encoding/encoding_mock.go -source=$GOPATH/src/$PACKAGE/src/dbnode/encoding/types.go"

src/dbnode/integration/bootstrap_retries_test.go

@@ -195,7 +195,7 @@ func TestNoOpenFilesWhenBootstrapRetriesDueToObsoleteRanges(t *testing.T) {
 		},
 	})
 
-	require.NoError(t, writeTestDataToDisk(ns1, setup, seriesMaps, 0))
+	require.NoError(t, writeTestDataToDiskWithIndex(ns1, setup, seriesMaps))
 	require.NoError(t, setup.StartServer()) // Blocks until bootstrap is complete
 	defer func() {
 		require.NoError(t, setup.StopServerAndVerifyOpenFilesAreClosed())

src/dbnode/integration/commitlog_bootstrap_coldwrites_test.go

@@ -85,7 +85,7 @@ func testCommitLogBootstrapColdWrites(t *testing.T, setTestOpts setTestOptions,
 		updateInputConfig(dataFilesData)
 	}
 	dataFilesSeriesMaps := generate.BlocksByStart(dataFilesData)
-	require.NoError(t, writeTestDataToDisk(ns1, setup, dataFilesSeriesMaps, 0))
+	require.NoError(t, writeTestDataToDiskWithIndex(ns1, setup, dataFilesSeriesMaps))
 	log.Info("finished writing data files")
 
 	log.Info("writing commit logs")

src/dbnode/integration/commitlog_bootstrap_merge_test.go

@@ -99,7 +99,7 @@ func TestCommitLogAndFSMergeBootstrap(t *testing.T) {
 		t0: seriesMaps[t0],
 		t1: seriesMaps[t1],
 	}
-	require.NoError(t, writeTestDataToDisk(ns1, setup, fsSeriesMaps, 0))
+	require.NoError(t, writeTestDataToDiskWithIndex(ns1, setup, fsSeriesMaps))
 
 	log.Info("writing commit logs")
 	commitlogSeriesMaps := generate.SeriesBlocksByStart{

src/dbnode/integration/fs_bootstrap_index_test.go

@@ -33,7 +33,9 @@ import (
 	"github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/dbnode/retention"
 	"github.com/m3db/m3/src/dbnode/storage/index"
+	"github.com/m3db/m3/src/m3ninx/doc"
 	"github.com/m3db/m3/src/m3ninx/idx"
+	idxpersist "github.com/m3db/m3/src/m3ninx/persist"
 	"github.com/m3db/m3/src/x/context"
 	"github.com/m3db/m3/src/x/ident"
 	xtime "github.com/m3db/m3/src/x/time"
@@ -119,16 +121,35 @@ func testFilesystemBootstrapIndexWithIndexingEnabled(
 		ID:   ident.StringID("foo"),
 		Tags: ident.NewTags(ident.StringTag("city", "new_york"), ident.StringTag("foo", "foo")),
 	}
+	fooDoc := doc.Metadata{
+		ID: fooSeries.ID.Bytes(),
+		Fields: []doc.Field{
+			{Name: []byte("city"), Value: []byte("new_york")},
+			{Name: []byte("foo"), Value: []byte("foo")},
+		},
+	}
 
 	barSeries := generate.Series{
 		ID:   ident.StringID("bar"),
 		Tags: ident.NewTags(ident.StringTag("city", "new_jersey")),
 	}
+	barDoc := doc.Metadata{
+		ID: barSeries.ID.Bytes(),
+		Fields: []doc.Field{
+			{Name: []byte("city"), Value: []byte("new_jersey")},
+		},
+	}
 
 	bazSeries := generate.Series{
 		ID:   ident.StringID("baz"),
 		Tags: ident.NewTags(ident.StringTag("city", "seattle")),
 	}
+	bazDoc := doc.Metadata{
+		ID: bazSeries.ID.Bytes(),
+		Fields: []doc.Field{
+			{Name: []byte("city"), Value: []byte("seattle")},
+		},
+	}
 
 	seriesMaps := generate.BlocksByStart([]generate.BlockConfig{
 		{
@@ -157,8 +178,22 @@ func testFilesystemBootstrapIndexWithIndexingEnabled(
 		},
 	})
 
+	defaultIndexDocs := []doc.Metadata{
+		fooDoc,
+		barDoc,
+		bazDoc,
+	}
+
 	require.NoError(t, writeTestDataToDisk(ns1, setup, seriesMaps, 0))
 	require.NoError(t, writeTestDataToDisk(ns2, setup, nil, 0))
+	require.NoError(t, writeTestIndexDataToDisk(
+		ns1,
+		setup.StorageOpts(),
+		idxpersist.DefaultIndexVolumeType,
+		now.Add(-blockSize),
+		setup.ShardSet().AllIDs(),
+		defaultIndexDocs,
+	))
 
 	// Start the server with filesystem bootstrapper
 	log := setup.StorageOpts().InstrumentOptions().Logger()

src/dbnode/integration/fs_bootstrap_multi_ns_test.go

@@ -76,8 +76,8 @@ func TestFilesystemBootstrapMultipleNamespaces(t *testing.T) {
 		{IDs: []string{"foo", "bar"}, NumPoints: 100, Start: now.Add(-ns2BlockSize)},
 		{IDs: []string{"foo", "baz"}, NumPoints: 50, Start: now},
 	})
-	require.NoError(t, writeTestDataToDisk(ns1, setup, ns1SeriesMaps, 0))
-	require.NoError(t, writeTestDataToDisk(ns2, setup, ns2SeriesMaps, 0))
+	require.NoError(t, writeTestDataToDiskWithIndex(ns1, setup, ns1SeriesMaps))
+	require.NoError(t, writeTestDataToDiskWithIndex(ns2, setup, ns2SeriesMaps))
 	log.Info("generated data")
 
 	// Start the server with filesystem bootstrapper

src/dbnode/integration/generate/generate.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/m3db/m3/src/dbnode/encoding/testgen"
 	"github.com/m3db/m3/src/dbnode/ts"
+	"github.com/m3db/m3/src/m3ninx/doc"
 	"github.com/m3db/m3/src/x/ident"
 	xtime "github.com/m3db/m3/src/x/time"
 )
@@ -105,6 +106,25 @@ func ToPointsByTime(seriesMaps SeriesBlocksByStart) SeriesDataPointsByTime {
 	return pointsByTime
 }
 
+// ToDocMetadata converts a SeriesBlock to []doc.Metadata
+func ToDocMetadata(seriesBlock SeriesBlock) []doc.Metadata {
+	docs := make([]doc.Metadata, 0)
+	for _, series := range seriesBlock {
+		fields := make([]doc.Field, 0)
+		for _, t := range series.Tags.Values() {
+			fields = append(fields, doc.Field{
+				Name:  t.Name.Bytes(),
+				Value: t.Value.Bytes(),
+			})
+		}
+		docs = append(docs, doc.Metadata{
+			ID:     series.ID.Bytes(),
+			Fields: fields,
+		})
+	}
+	return docs
+}
+
 // Dearrange de-arranges the list by the defined percent.
 func (l SeriesDataPointsByTime) Dearrange(percent float64) SeriesDataPointsByTime {
 	numDis := percent * float64(len(l))