diff --git a/src/dbnode/storage/index/block_test.go b/src/dbnode/storage/index/block_test.go index 67f8710f9e..f84914962f 100644 --- a/src/dbnode/storage/index/block_test.go +++ b/src/dbnode/storage/index/block_test.go @@ -452,8 +452,8 @@ func TestBlockQueryAddResultsSegmentsError(t *testing.T) { b.mutableSegments.foregroundSegments = []*readableSeg{newReadableSeg(seg1, testOpts)} b.shardRangesSegmentsByVolumeType = map[idxpersist.IndexVolumeType][]blockShardRangesSegments{ - idxpersist.DefaultIndexVolumeType: []blockShardRangesSegments{ - blockShardRangesSegments{segments: []segment.Segment{seg2, seg3}}, + idxpersist.DefaultIndexVolumeType: { + {segments: []segment.Segment{seg2, seg3}}, }, } @@ -518,7 +518,7 @@ func TestBlockMockQueryExecutorExecIterErr(t *testing.T) { return exec, nil } - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -559,7 +559,7 @@ func TestBlockMockQueryExecutorExecLimit(t *testing.T) { return exec, nil } - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -610,7 +610,7 @@ func TestBlockMockQueryExecutorExecIterCloseErr(t *testing.T) { return exec, nil } - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(false), @@ -649,7 +649,7 @@ func TestBlockMockQuerySeriesLimitNonExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -699,7 +699,7 @@ func TestBlockMockQuerySeriesLimitExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -751,7 +751,7 @@ func TestBlockMockQueryDocsLimitNonExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -801,7 +801,7 @@ func TestBlockMockQueryDocsLimitExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -860,7 +860,7 @@ func TestBlockMockQueryMergeResultsMapLimit(t *testing.T) { _, _, err = results.AddDocuments([]doc.Metadata{testDoc1()}) require.NoError(t, err) - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -911,7 +911,7 @@ func TestBlockMockQueryMergeResultsDupeID(t *testing.T) { _, _, err = results.AddDocuments([]doc.Metadata{testDoc1()}) require.NoError(t, err) - dIter := doc.NewMockIterator(ctrl) + dIter := doc.NewMockMetadataIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -1900,8 +1900,8 @@ func TestBlockAggregate(t *testing.T) { require.True(t, exhaustive) assertAggregateResultsMapEquals(t, map[string][]string{ - "f1": []string{"t1", "t2", "t3"}, - "f2": []string{"t1"}, + "f1": {"t1", "t2", "t3"}, + "f2": {"t1"}, }, results) sp.Finish() @@ -1976,7 +1976,7 @@ func TestBlockAggregateNotExhaustive(t *testing.T) { require.False(t, exhaustive) assertAggregateResultsMapEquals(t, map[string][]string{ - "f1": []string{"t1"}, + "f1": {"t1"}, }, results) sp.Finish() @@ -2067,8 +2067,8 @@ func TestBlockE2EInsertAggregate(t *testing.T) { require.NoError(t, err) require.True(t, exhaustive) assertAggregateResultsMapEquals(t, map[string][]string{ - "bar": []string{"baz", "qux"}, - "some": []string{"more", "other"}, + "bar": {"baz", "qux"}, + "some": {"more", "other"}, }, results) results = NewAggregateResults(ident.StringID("ns"), AggregateResultsOptions{ @@ -2085,7 +2085,7 @@ func TestBlockE2EInsertAggregate(t *testing.T) { require.NoError(t, err) require.True(t, exhaustive) assertAggregateResultsMapEquals(t, map[string][]string{ - "bar": []string{"baz", "qux"}, + "bar": {"baz", "qux"}, }, results) results = NewAggregateResults(ident.StringID("ns"), AggregateResultsOptions{ @@ -2162,7 +2162,7 @@ func testDoc1() doc.Metadata { return doc.Metadata{ ID: []byte("foo"), Fields: []doc.Field{ - doc.Field{ + { Name: []byte("bar"), Value: []byte("baz"), }, @@ -2174,11 +2174,11 @@ func testDoc1DupeID() doc.Metadata { return doc.Metadata{ ID: []byte("foo"), Fields: []doc.Field{ - doc.Field{ + { Name: []byte("why"), Value: []byte("not"), }, - doc.Field{ + { Name: []byte("some"), Value: []byte("more"), }, @@ -2190,11 +2190,11 @@ func testDoc2() doc.Metadata { return doc.Metadata{ ID: []byte("something"), Fields: []doc.Field{ - doc.Field{ + { Name: []byte("bar"), Value: []byte("baz"), }, - doc.Field{ + { Name: []byte("some"), Value: []byte("more"), }, @@ -2206,11 +2206,11 @@ func testDoc3() doc.Metadata { return doc.Metadata{ ID: []byte("bar"), Fields: []doc.Field{ - doc.Field{ + { Name: []byte("bar"), Value: []byte("qux"), }, - doc.Field{ + { Name: []byte("some"), Value: []byte("other"), }, diff --git a/src/dbnode/storage/index/read_through_segment.go b/src/dbnode/storage/index/read_through_segment.go index 2de3fb368c..089fd546bc 100644 --- a/src/dbnode/storage/index/read_through_segment.go +++ b/src/dbnode/storage/index/read_through_segment.go @@ -262,8 +262,18 @@ func (s *readThroughSegmentReader) AllDocs() (index.IDDocIterator, error) { return s.reader.AllDocs() } +// Metadata is a pass through call, since there's no postings list to cache. +func (s *readThroughSegmentReader) Metadata(id postings.ID) (doc.Metadata, error) { + return s.reader.Metadata(id) +} + +// MetadataIterator is a pass through call, since there's no postings list to cache. +func (s *readThroughSegmentReader) MetadataIterator(pl postings.List) (doc.MetadataIterator, error) { + return s.reader.MetadataIterator(pl) +} + // Doc is a pass through call, since there's no postings list to cache. -func (s *readThroughSegmentReader) Doc(id postings.ID) (doc.Metadata, error) { +func (s *readThroughSegmentReader) Doc(id postings.ID) (doc.Document, error) { return s.reader.Doc(id) } diff --git a/src/m3ninx/doc/doc_mock.go b/src/m3ninx/doc/doc_mock.go index 25e54975a1..d955067ed1 100644 --- a/src/m3ninx/doc/doc_mock.go +++ b/src/m3ninx/doc/doc_mock.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/m3db/m3/src/m3ninx/doc/types.go -// Copyright (c) 2018 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -30,6 +30,85 @@ import ( "github.com/golang/mock/gomock" ) +// MockMetadataIterator is a mock of MetadataIterator interface +type MockMetadataIterator struct { + ctrl *gomock.Controller + recorder *MockMetadataIteratorMockRecorder +} + +// MockMetadataIteratorMockRecorder is the mock recorder for MockMetadataIterator +type MockMetadataIteratorMockRecorder struct { + mock *MockMetadataIterator +} + +// NewMockMetadataIterator creates a new mock instance +func NewMockMetadataIterator(ctrl *gomock.Controller) *MockMetadataIterator { + mock := &MockMetadataIterator{ctrl: ctrl} + mock.recorder = &MockMetadataIteratorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockMetadataIterator) EXPECT() *MockMetadataIteratorMockRecorder { + return m.recorder +} + +// Next mocks base method +func (m *MockMetadataIterator) Next() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Next indicates an expected call of Next +func (mr *MockMetadataIteratorMockRecorder) Next() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Next", reflect.TypeOf((*MockMetadataIterator)(nil).Next)) +} + +// Current mocks base method +func (m *MockMetadataIterator) Current() Metadata { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Current") + ret0, _ := ret[0].(Metadata) + return ret0 +} + +// Current indicates an expected call of Current +func (mr *MockMetadataIteratorMockRecorder) Current() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Current", reflect.TypeOf((*MockMetadataIterator)(nil).Current)) +} + +// Err mocks base method +func (m *MockMetadataIterator) Err() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +// Err indicates an expected call of Err +func (mr *MockMetadataIteratorMockRecorder) Err() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Err", reflect.TypeOf((*MockMetadataIterator)(nil).Err)) +} + +// Close mocks base method +func (m *MockMetadataIterator) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close +func (mr *MockMetadataIteratorMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockMetadataIterator)(nil).Close)) +} + // MockIterator is a mock of Iterator interface type MockIterator struct { ctrl *gomock.Controller @@ -68,10 +147,10 @@ func (mr *MockIteratorMockRecorder) Next() *gomock.Call { } // Current mocks base method -func (m *MockIterator) Current() Metadata { +func (m *MockIterator) Current() Document { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Current") - ret0, _ := ret[0].(Metadata) + ret0, _ := ret[0].(Document) return ret0 } diff --git a/src/m3ninx/doc/document.go b/src/m3ninx/doc/document.go index bb92d6525d..7dc3f04a36 100644 --- a/src/m3ninx/doc/document.go +++ b/src/m3ninx/doc/document.go @@ -212,3 +212,47 @@ func (ds Documents) Less(i, j int) bool { func (ds Documents) Swap(i, j int) { ds[i], ds[j] = ds[j], ds[i] } + +// Encoded is a serialized document metadata. +type Encoded struct { + Bytes []byte +} + +// Document contains either metadata or an encoded metadata +// but never both. +type Document struct { + metadata Metadata + encoded Encoded + + hasMetadata bool +} + +// NewDocumentFromMetadata creates a Document from a Metadata. +func NewDocumentFromMetadata(m Metadata) Document { + return Document{metadata: m, hasMetadata: true} +} + +// NewDocumentFromEncoded creates a Document from an Encoded. +func NewDocumentFromEncoded(e Encoded) Document { + return Document{encoded: e} +} + +// Metadata returns the metadata it contains, if it has one. Otherwise returns an empty metadata +// and false. +func (d *Document) Metadata() (Metadata, bool) { + if d.hasMetadata { + return d.metadata, true + } + + return Metadata{}, false +} + +// Encoded returns the encoded metadata it contains, if it has one. Otherwise returns an +// empty encoded metadata and false. +func (d *Document) Encoded() (Encoded, bool) { + if !d.hasMetadata { + return d.encoded, true + } + + return Encoded{}, false +} diff --git a/src/m3ninx/doc/types.go b/src/m3ninx/doc/types.go index 5593b07f24..1eb41a0992 100644 --- a/src/m3ninx/doc/types.go +++ b/src/m3ninx/doc/types.go @@ -20,14 +20,14 @@ package doc -// Iterator provides an iterator over a collection of documents. It is NOT safe for multiple -// goroutines to invoke methods on an Iterator simultaneously. -type Iterator interface { - // Next returns a bool indicating if the iterator has any more documents +// MetadataIterator provides an iterator over a collection of document metadata. It is NOT +// safe for multiple goroutines to invoke methods on an MetadataIterator simultaneously. +type MetadataIterator interface { + // Next returns a bool indicating if the iterator has any more metadata // to return. Next() bool - // Current returns the current document. It is only safe to call Current immediately + // Current returns the current metadata. It is only safe to call Current immediately // after a call to Next confirms there are more elements remaining. The Metadata // returned from Current is only valid until the following call to Next(). Callers // should copy the Metadata if they need it live longer. @@ -39,3 +39,23 @@ type Iterator interface { // Close releases any internal resources used by the iterator. Close() error } + +// Iterator provides an iterator over a collection of documents. It is NOT +// safe for multiple goroutines to invoke methods on an Iterator simultaneously. +type Iterator interface { + // Next returns a bool indicating if the iterator has any more documents + // to return. + Next() bool + + // Current returns the current document. It is only safe to call Current immediately + // after a call to Next confirms there are more elements remaining. The Document + // returned from Current is only valid until the following call to Next(). Callers + // should copy the Document if they need it live longer. + Current() Document + + // Err returns any errors encountered during iteration. + Err() error + + // Close releases any internal resources used by the iterator. + Close() error +} diff --git a/src/m3ninx/generated/mocks/generate.go b/src/m3ninx/generated/mocks/generate.go index 33756ac9a4..418a43997c 100644 --- a/src/m3ninx/generated/mocks/generate.go +++ b/src/m3ninx/generated/mocks/generate.go @@ -30,4 +30,4 @@ package mocks // mockgen rules for generating mocks (reflection mode) //go:generate sh -c "mockgen -package=mem -destination=$GOPATH/src/github.com/m3db/m3/src/m3ninx/index/segment/mem/mem_mock.go github.com/m3db/m3/src/m3ninx/index/segment/mem ReadableSegment" //go:generate sh -c "mockgen -package=fst -destination=$GOPATH/src/github.com/m3db/m3/src/m3ninx/index/segment/fst/fst_mock.go github.com/m3db/m3/src/m3ninx/index/segment/fst Writer,Segment" -//go:generate sh -c "mockgen -package=index -destination=$GOPATH/src/github.com/m3db/m3/src/m3ninx/index/index_mock.go github.com/m3db/m3/src/m3ninx/index Reader,DocRetriever" +//go:generate sh -c "mockgen -package=index -destination=$GOPATH/src/github.com/m3db/m3/src/m3ninx/index/index_mock.go github.com/m3db/m3/src/m3ninx/index Reader,DocRetriever,MetadataRetriever" diff --git a/src/m3ninx/idx/types.go b/src/m3ninx/idx/types.go index 11fe2609ad..ceee1cbfe5 100644 --- a/src/m3ninx/idx/types.go +++ b/src/m3ninx/idx/types.go @@ -38,7 +38,7 @@ type Index interface { // Searcher provides search over a point-in-time view of an index. type Searcher interface { - Search(q Query) (doc.Iterator, error) + Search(q Query) (doc.MetadataIterator, error) Close() error } diff --git a/src/m3ninx/index/index_mock.go b/src/m3ninx/index/index_mock.go index 4888262daf..d7899331dd 100644 --- a/src/m3ninx/index/index_mock.go +++ b/src/m3ninx/index/index_mock.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/m3db/m3/src/m3ninx/index (interfaces: Reader,DocRetriever) +// Source: github.com/m3db/m3/src/m3ninx/index (interfaces: Reader,DocRetriever,MetadataRetriever) -// Copyright (c) 2019 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -86,10 +86,10 @@ func (mr *MockReaderMockRecorder) Close() *gomock.Call { } // Doc mocks base method -func (m *MockReader) Doc(arg0 postings.ID) (doc.Metadata, error) { +func (m *MockReader) Doc(arg0 postings.ID) (doc.Document, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Doc", arg0) - ret0, _ := ret[0].(doc.Metadata) + ret0, _ := ret[0].(doc.Document) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -175,6 +175,36 @@ func (mr *MockReaderMockRecorder) MatchTerm(arg0, arg1 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchTerm", reflect.TypeOf((*MockReader)(nil).MatchTerm), arg0, arg1) } +// Metadata mocks base method +func (m *MockReader) Metadata(arg0 postings.ID) (doc.Metadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Metadata", arg0) + ret0, _ := ret[0].(doc.Metadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Metadata indicates an expected call of Metadata +func (mr *MockReaderMockRecorder) Metadata(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Metadata", reflect.TypeOf((*MockReader)(nil).Metadata), arg0) +} + +// MetadataIterator mocks base method +func (m *MockReader) MetadataIterator(arg0 postings.List) (doc.MetadataIterator, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MetadataIterator", arg0) + ret0, _ := ret[0].(doc.MetadataIterator) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MetadataIterator indicates an expected call of MetadataIterator +func (mr *MockReaderMockRecorder) MetadataIterator(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MetadataIterator", reflect.TypeOf((*MockReader)(nil).MetadataIterator), arg0) +} + // MockDocRetriever is a mock of DocRetriever interface type MockDocRetriever struct { ctrl *gomock.Controller @@ -199,10 +229,10 @@ func (m *MockDocRetriever) EXPECT() *MockDocRetrieverMockRecorder { } // Doc mocks base method -func (m *MockDocRetriever) Doc(arg0 postings.ID) (doc.Metadata, error) { +func (m *MockDocRetriever) Doc(arg0 postings.ID) (doc.Document, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Doc", arg0) - ret0, _ := ret[0].(doc.Metadata) + ret0, _ := ret[0].(doc.Document) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -212,3 +242,41 @@ func (mr *MockDocRetrieverMockRecorder) Doc(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Doc", reflect.TypeOf((*MockDocRetriever)(nil).Doc), arg0) } + +// MockMetadataRetriever is a mock of MetadataRetriever interface +type MockMetadataRetriever struct { + ctrl *gomock.Controller + recorder *MockMetadataRetrieverMockRecorder +} + +// MockMetadataRetrieverMockRecorder is the mock recorder for MockMetadataRetriever +type MockMetadataRetrieverMockRecorder struct { + mock *MockMetadataRetriever +} + +// NewMockMetadataRetriever creates a new mock instance +func NewMockMetadataRetriever(ctrl *gomock.Controller) *MockMetadataRetriever { + mock := &MockMetadataRetriever{ctrl: ctrl} + mock.recorder = &MockMetadataRetrieverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockMetadataRetriever) EXPECT() *MockMetadataRetrieverMockRecorder { + return m.recorder +} + +// Metadata mocks base method +func (m *MockMetadataRetriever) Metadata(arg0 postings.ID) (doc.Metadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Metadata", arg0) + ret0, _ := ret[0].(doc.Metadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Metadata indicates an expected call of Metadata +func (mr *MockMetadataRetrieverMockRecorder) Metadata(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Metadata", reflect.TypeOf((*MockMetadataRetriever)(nil).Metadata), arg0) +} diff --git a/src/m3ninx/index/iterator.go b/src/m3ninx/index/iterator.go index 464e430517..b41bdf6735 100644 --- a/src/m3ninx/index/iterator.go +++ b/src/m3ninx/index/iterator.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,69 +21,59 @@ package index import ( - "errors" - "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/m3ninx/postings" ) -var ( - errIteratorClosed = errors.New("iterator has been closed") -) - -type idDocIterator struct { +type documentIterator struct { retriever DocRetriever postingsIter postings.Iterator - currDoc doc.Metadata + currDoc doc.Document currID postings.ID closed bool err error } -// NewIDDocIterator returns a new NewIDDocIterator. -func NewIDDocIterator(r DocRetriever, pi postings.Iterator) IDDocIterator { - return &idDocIterator{ +// NewIterator returns a new Iterator +func NewIterator(r DocRetriever, pi postings.Iterator) doc.Iterator { + return &documentIterator{ retriever: r, postingsIter: pi, } } -func (it *idDocIterator) Next() bool { - if it.closed || it.err != nil || !it.postingsIter.Next() { +func (e *documentIterator) Next() bool { + if e.closed || e.err != nil || !e.postingsIter.Next() { return false } - id := it.postingsIter.Current() - it.currID = id + id := e.postingsIter.Current() + e.currID = id - d, err := it.retriever.Doc(id) + d, err := e.retriever.Doc(id) if err != nil { - it.err = err + e.err = err return false } - it.currDoc = d + e.currDoc = d return true } -func (it *idDocIterator) Current() doc.Metadata { - return it.currDoc -} - -func (it *idDocIterator) PostingsID() postings.ID { - return it.currID +func (e *documentIterator) Current() doc.Document { + return e.currDoc } -func (it *idDocIterator) Err() error { - return it.err +func (e *documentIterator) Err() error { + return e.err } -func (it *idDocIterator) Close() error { - if it.closed { +func (e *documentIterator) Close() error { + if e.closed { return errIteratorClosed } - it.closed = true - it.currDoc = doc.Metadata{} - it.currID = postings.ID(0) - err := it.postingsIter.Close() + e.closed = true + e.currDoc = doc.Document{} + e.currID = postings.ID(0) + err := e.postingsIter.Close() return err } diff --git a/src/m3ninx/index/iterator_test.go b/src/m3ninx/index/iterator_test.go index d0da569c2f..8083ad04b3 100644 --- a/src/m3ninx/index/iterator_test.go +++ b/src/m3ninx/index/iterator_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,88 +24,89 @@ import ( "testing" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding" "github.com/m3db/m3/src/m3ninx/postings" + xtest "github.com/m3db/m3/src/x/test" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) -func TestIterator(t *testing.T) { - mockCtrl := gomock.NewController(t) +func TestDocIterator(t *testing.T) { + mockCtrl := xtest.NewController(t) defer mockCtrl.Finish() - docWithIds := []struct { - id postings.ID - doc doc.Metadata - }{ + docs := []doc.Metadata{ { - id: 42, - doc: doc.Metadata{ - Fields: []doc.Field{ - doc.Field{ - Name: []byte("apple"), - Value: []byte("red"), - }, + ID: []byte("doc-id-1"), + Fields: []doc.Field{ + { + Name: []byte("apple"), + Value: []byte("red"), }, }, }, { - id: 53, - doc: doc.Metadata{ - Fields: []doc.Field{ - doc.Field{ - Name: []byte("banana"), - Value: []byte("yellow"), - }, - }, - }, - }, - { - id: 81, - doc: doc.Metadata{ - Fields: []doc.Field{ - doc.Field{ - Name: []byte("carrot"), - Value: []byte("orange"), - }, + ID: []byte("doc-id-2"), + Fields: []doc.Field{ + { + Name: []byte("banana"), + Value: []byte("yellow"), }, }, }, } + encodedDocsWithIds := make([]docsWithIDs, 0, len(docs)) + for i, d := range docs { + encodedDocsWithIds = append(encodedDocsWithIds, docsWithIDs{ + id: postings.ID(i), + doc: doc.NewDocumentFromEncoded( + doc.Encoded{Bytes: docToBytes(d)}), + }) + } + retriever := NewMockDocRetriever(mockCtrl) gomock.InOrder( - retriever.EXPECT().Doc(docWithIds[0].id).Return(docWithIds[0].doc, nil), - retriever.EXPECT().Doc(docWithIds[1].id).Return(docWithIds[1].doc, nil), - retriever.EXPECT().Doc(docWithIds[2].id).Return(docWithIds[2].doc, nil), + retriever.EXPECT().Doc(encodedDocsWithIds[0].id).Return(encodedDocsWithIds[0].doc, nil), + retriever.EXPECT().Doc(encodedDocsWithIds[1].id).Return(encodedDocsWithIds[1].doc, nil), ) postingsIter := postings.NewMockIterator(mockCtrl) gomock.InOrder( postingsIter.EXPECT().Next().Return(true), - postingsIter.EXPECT().Current().Return(docWithIds[0].id), + postingsIter.EXPECT().Current().Return(encodedDocsWithIds[0].id), postingsIter.EXPECT().Next().Return(true), - postingsIter.EXPECT().Current().Return(docWithIds[1].id), - postingsIter.EXPECT().Next().Return(true), - postingsIter.EXPECT().Current().Return(docWithIds[2].id), + postingsIter.EXPECT().Current().Return(encodedDocsWithIds[1].id), postingsIter.EXPECT().Next().Return(false), postingsIter.EXPECT().Close().Return(nil), ) - it := NewIDDocIterator(retriever, postingsIter) + it := NewIterator(retriever, postingsIter) require.True(t, it.Next()) - require.Equal(t, docWithIds[0].doc, it.Current()) - require.Equal(t, docWithIds[0].id, it.PostingsID()) - require.True(t, it.Next()) - require.Equal(t, docWithIds[1].doc, it.Current()) - require.Equal(t, docWithIds[1].id, it.PostingsID()) + require.Equal(t, encodedDocsWithIds[0].doc, it.Current()) require.True(t, it.Next()) - require.Equal(t, docWithIds[2].doc, it.Current()) - require.Equal(t, docWithIds[2].id, it.PostingsID()) - + require.Equal(t, encodedDocsWithIds[1].doc, it.Current()) require.False(t, it.Next()) require.NoError(t, it.Err()) require.NoError(t, it.Close()) } + +type docsWithIDs struct { + id postings.ID + doc doc.Document +} + +func docToBytes(d doc.Metadata) []byte { + enc := encoding.NewEncoder(1024) + n := enc.PutBytes(d.ID) + n += enc.PutUvarint(uint64(len(d.Fields))) + for _, f := range d.Fields { // nolint:gocritic + n += enc.PutBytes(f.Name) + n += enc.PutBytes(f.Value) + } + + return enc.Bytes() +} diff --git a/src/m3ninx/index/metadata_iterator.go b/src/m3ninx/index/metadata_iterator.go new file mode 100644 index 0000000000..afe53d9ecd --- /dev/null +++ b/src/m3ninx/index/metadata_iterator.go @@ -0,0 +1,87 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package index + +import ( + "errors" + + "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/postings" +) + +var errIteratorClosed = errors.New("iterator has been closed") + +type idDocIterator struct { + retriever MetadataRetriever + postingsIter postings.Iterator + + currDoc doc.Metadata + currID postings.ID + closed bool + err error +} + +// NewIDDocIterator returns a new NewIDDocIterator. +func NewIDDocIterator(r MetadataRetriever, pi postings.Iterator) IDDocIterator { + return &idDocIterator{ + retriever: r, + postingsIter: pi, + } +} + +func (it *idDocIterator) Next() bool { + if it.closed || it.err != nil || !it.postingsIter.Next() { + return false + } + id := it.postingsIter.Current() + it.currID = id + + d, err := it.retriever.Metadata(id) + if err != nil { + it.err = err + return false + } + it.currDoc = d + return true +} + +func (it *idDocIterator) Current() doc.Metadata { + return it.currDoc +} + +func (it *idDocIterator) PostingsID() postings.ID { + return it.currID +} + +func (it *idDocIterator) Err() error { + return it.err +} + +func (it *idDocIterator) Close() error { + if it.closed { + return errIteratorClosed + } + it.closed = true + it.currDoc = doc.Metadata{} + it.currID = postings.ID(0) + err := it.postingsIter.Close() + return err +} diff --git a/src/m3ninx/index/metadata_iterator_test.go b/src/m3ninx/index/metadata_iterator_test.go new file mode 100644 index 0000000000..2e17ebe252 --- /dev/null +++ b/src/m3ninx/index/metadata_iterator_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package index + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/postings" +) + +func TestIterator(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + docWithIds := []struct { + id postings.ID + doc doc.Metadata + }{ + { + id: 42, + doc: doc.Metadata{ + Fields: []doc.Field{ + { + Name: []byte("apple"), + Value: []byte("red"), + }, + }, + }, + }, + { + id: 53, + doc: doc.Metadata{ + Fields: []doc.Field{ + { + Name: []byte("banana"), + Value: []byte("yellow"), + }, + }, + }, + }, + { + id: 81, + doc: doc.Metadata{ + Fields: []doc.Field{ + { + Name: []byte("carrot"), + Value: []byte("orange"), + }, + }, + }, + }, + } + + retriever := NewMockMetadataRetriever(mockCtrl) + gomock.InOrder( + retriever.EXPECT().Metadata(docWithIds[0].id).Return(docWithIds[0].doc, nil), + retriever.EXPECT().Metadata(docWithIds[1].id).Return(docWithIds[1].doc, nil), + retriever.EXPECT().Metadata(docWithIds[2].id).Return(docWithIds[2].doc, nil), + ) + + postingsIter := postings.NewMockIterator(mockCtrl) + gomock.InOrder( + postingsIter.EXPECT().Next().Return(true), + postingsIter.EXPECT().Current().Return(docWithIds[0].id), + postingsIter.EXPECT().Next().Return(true), + postingsIter.EXPECT().Current().Return(docWithIds[1].id), + postingsIter.EXPECT().Next().Return(true), + postingsIter.EXPECT().Current().Return(docWithIds[2].id), + postingsIter.EXPECT().Next().Return(false), + postingsIter.EXPECT().Close().Return(nil), + ) + + it := NewIDDocIterator(retriever, postingsIter) + + require.True(t, it.Next()) + require.Equal(t, docWithIds[0].doc, it.Current()) + require.Equal(t, docWithIds[0].id, it.PostingsID()) + require.True(t, it.Next()) + require.Equal(t, docWithIds[1].doc, it.Current()) + require.Equal(t, docWithIds[1].id, it.PostingsID()) + require.True(t, it.Next()) + require.Equal(t, docWithIds[2].doc, it.Current()) + require.Equal(t, docWithIds[2].id, it.PostingsID()) + + require.False(t, it.Next()) + require.NoError(t, it.Err()) + + require.NoError(t, it.Close()) +} diff --git a/src/m3ninx/index/segment/builder/builder.go b/src/m3ninx/index/segment/builder/builder.go index 4e2576b6f0..6217d8e820 100644 --- a/src/m3ninx/index/segment/builder/builder.go +++ b/src/m3ninx/index/segment/builder/builder.go @@ -485,7 +485,7 @@ func (b *builder) AllDocs() (index.IDDocIterator, error) { return index.NewIDDocIterator(b, rangeIter), nil } -func (b *builder) Doc(id postings.ID) (doc.Metadata, error) { +func (b *builder) Metadata(id postings.ID) (doc.Metadata, error) { b.status.RLock() defer b.status.RUnlock() diff --git a/src/m3ninx/index/segment/builder/multi_segments_builder.go b/src/m3ninx/index/segment/builder/multi_segments_builder.go index 4c156e5127..7c8822b711 100644 --- a/src/m3ninx/index/segment/builder/multi_segments_builder.go +++ b/src/m3ninx/index/segment/builder/multi_segments_builder.go @@ -159,7 +159,7 @@ func (b *builderFromSegments) AllDocs() (index.IDDocIterator, error) { return index.NewIDDocIterator(b, rangeIter), nil } -func (b *builderFromSegments) Doc(id postings.ID) (doc.Metadata, error) { +func (b *builderFromSegments) Metadata(id postings.ID) (doc.Metadata, error) { idx := int(id) if idx < 0 || idx >= len(b.docs) { return doc.Metadata{}, errDocNotFound diff --git a/src/m3ninx/index/segment/fst/encoding/docs/data.go b/src/m3ninx/index/segment/fst/encoding/docs/data.go index b702cfacfb..00092152b3 100644 --- a/src/m3ninx/index/segment/fst/encoding/docs/data.go +++ b/src/m3ninx/index/segment/fst/encoding/docs/data.go @@ -21,6 +21,7 @@ package docs import ( + "errors" "fmt" "io" @@ -128,3 +129,112 @@ func (r *DataReader) Read(offset uint64) (doc.Metadata, error) { return d, nil } + +// EncodedDataReader is a reader for the data file for encoded document metadata. +type EncodedDataReader struct { + data []byte +} + +// NewEncodedDataReader returns a new EncodedDataReader. +func NewEncodedDataReader(data []byte) *EncodedDataReader { + return &EncodedDataReader{ + data: data, + } +} + +// Read reads a doc.Encoded from a data stream starting at the specified offset. +func (e *EncodedDataReader) Read(offset uint64) (doc.Encoded, error) { + if offset >= uint64(len(e.data)) { + return doc.Encoded{}, fmt.Errorf( + "invalid offset: %v is past the end of the data file", offset, + ) + } + + return doc.Encoded{ + Bytes: e.data[int(offset):], + }, nil +} + +// EncodedDocumentReader is a reader for reading documents from encoded metadata. +type EncodedDocumentReader struct { + currFields []doc.Field +} + +// NewEncodedDocumentReader returns a new EncodedDocumentReader. +func NewEncodedDocumentReader() *EncodedDocumentReader { + return &EncodedDocumentReader{} +} + +// Read reads a doc.Metadata from a doc.Encoded. Returned doc.Metadata should be +// processed before calling Read again as the underlying array pointed to by the Fields +// slice will be updated. This approach avoids allocating a new slice with a new backing +// array for every document processed, unlike (*DataReader).Read +func (r *EncodedDocumentReader) Read(encoded doc.Encoded) (doc.Metadata, error) { + for i := range r.currFields { + r.currFields[i] = doc.Field{} + } + r.currFields = r.currFields[:0] + id, buf, err := encoding.ReadBytes(encoded.Bytes) + if err != nil { + return doc.Metadata{}, err + } + + x, buf, err := encoding.ReadUvarint(buf) + if err != nil { + return doc.Metadata{}, err + } + n := int(x) + + var name, val []byte + for i := 0; i < n; i++ { + name, buf, err = encoding.ReadBytes(buf) + if err != nil { + return doc.Metadata{}, err + } + val, buf, err = encoding.ReadBytes(buf) + if err != nil { + return doc.Metadata{}, err + } + r.currFields = append(r.currFields, doc.Field{ + Name: name, + Value: val, + }) + } + + return doc.Metadata{ + ID: id, + Fields: r.currFields, + }, nil +} + +// ReadEncodedDocumentID reads the document ID from the encoded document metadata. +func ReadEncodedDocumentID(encoded doc.Encoded) ([]byte, error) { + id, _, err := encoding.ReadBytes(encoded.Bytes) + return id, err +} + +// GetFromDocument retrieves a doc.Metadata from a doc.Document. +func GetFromDocument(document doc.Document, reader *EncodedDocumentReader) (doc.Metadata, error) { + if d, ok := document.Metadata(); ok { + return d, nil + } + + if e, ok := document.Encoded(); ok { + return reader.Read(e) + } + + return doc.Metadata{}, errors.New("document does not contain metadata or encoded metadata") +} + +// ReadIDFromDocument reads the document ID from the document. +func ReadIDFromDocument(document doc.Document) ([]byte, error) { + if d, ok := document.Metadata(); ok { + return d.ID, nil + } + + if e, ok := document.Encoded(); ok { + return ReadEncodedDocumentID(e) + } + + return nil, errors.New("document does not contain metadata or encoded metadata") +} diff --git a/src/m3ninx/index/segment/fst/encoding/docs/data_test.go b/src/m3ninx/index/segment/fst/encoding/docs/data_test.go index 1257b406c6..792c107680 100644 --- a/src/m3ninx/index/segment/fst/encoding/docs/data_test.go +++ b/src/m3ninx/index/segment/fst/encoding/docs/data_test.go @@ -30,56 +30,56 @@ import ( "github.com/stretchr/testify/require" ) -func TestStoredFieldsData(t *testing.T) { - tests := []struct { - name string - docs []doc.Metadata - }{ - { - name: "empty document", - docs: []doc.Metadata{ - { - Fields: doc.Fields{}, - }, +var tests = []struct { + name string + docs []doc.Metadata +}{ + { + name: "empty document", + docs: []doc.Metadata{ + { + Fields: doc.Fields{}, }, }, - { - name: "standard documents", - docs: []doc.Metadata{ - { - ID: []byte("831992"), - Fields: []doc.Field{ - { - Name: []byte("fruit"), - Value: []byte("apple"), - }, - { - Name: []byte("color"), - Value: []byte("red"), - }, + }, + { + name: "standard documents", + docs: []doc.Metadata{ + { + ID: []byte("831992"), + Fields: []doc.Field{ + { + Name: []byte("fruit"), + Value: []byte("apple"), + }, + { + Name: []byte("color"), + Value: []byte("red"), }, }, - { - ID: []byte("080392"), - Fields: []doc.Field{ - { - Name: []byte("fruit"), - Value: []byte("banana"), - }, - { - Name: []byte("color"), - Value: []byte("yellow"), - }, + }, + { + ID: []byte("080392"), + Fields: []doc.Field{ + { + Name: []byte("fruit"), + Value: []byte("banana"), + }, + { + Name: []byte("color"), + Value: []byte("yellow"), }, }, }, }, - { - name: "node exporter metrics", - docs: util.MustReadDocs("../../../../../util/testdata/node_exporter.json", 2000), - }, - } + }, + { + name: "node exporter metrics", + docs: util.MustReadDocs("../../../../../util/testdata/node_exporter.json", 2000), + }, +} +func TestStoredFieldsData(t *testing.T) { w := NewDataWriter(nil) for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -106,3 +106,35 @@ func TestStoredFieldsData(t *testing.T) { }) } } + +func TestEncodedDataReader(t *testing.T) { + w := NewDataWriter(nil) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var ( + buf = new(bytes.Buffer) + offsets = make([]int, 0) + idx int + ) + w.Reset(buf) + + for i := range test.docs { + n, err := w.Write(test.docs[i]) + require.NoError(t, err) + offsets = append(offsets, idx) + idx += n + } + + dataReader := NewEncodedDataReader(buf.Bytes()) + docReader := NewEncodedDocumentReader() + for i := range test.docs { + encoded, err := dataReader.Read(uint64(offsets[i])) + require.NoError(t, err) + + actual, err := docReader.Read(encoded) + require.NoError(t, err) + require.True(t, actual.Equal(test.docs[i])) + } + }) + } +} diff --git a/src/m3ninx/index/segment/fst/encoding/docs/slice.go b/src/m3ninx/index/segment/fst/encoding/docs/slice.go index 20df8c8a5e..02111d6fa1 100644 --- a/src/m3ninx/index/segment/fst/encoding/docs/slice.go +++ b/src/m3ninx/index/segment/fst/encoding/docs/slice.go @@ -33,7 +33,7 @@ var ( ) var _ Reader = (*SliceReader)(nil) -var _ index.DocRetriever = (*SliceReader)(nil) +var _ index.MetadataRetriever = (*SliceReader)(nil) // SliceReader is a docs slice reader for use with documents // stored in memory. @@ -61,8 +61,8 @@ func (r *SliceReader) Read(id postings.ID) (doc.Metadata, error) { return r.docs[idx], nil } -// Doc implements DocRetriever and reads the document with postings ID. -func (r *SliceReader) Doc(id postings.ID) (doc.Metadata, error) { +// Metadata implements MetadataRetriever and reads the document with postings ID. +func (r *SliceReader) Metadata(id postings.ID) (doc.Metadata, error) { return r.Read(id) } diff --git a/src/m3ninx/index/segment/fst/encoding/encoding.go b/src/m3ninx/index/segment/fst/encoding/encoding.go index 09fbb23e4f..9c0f35a809 100644 --- a/src/m3ninx/index/segment/fst/encoding/encoding.go +++ b/src/m3ninx/index/segment/fst/encoding/encoding.go @@ -117,34 +117,57 @@ func (d *Decoder) Uint64() (uint64, error) { // Uvarint reads a variable-sized unsigned integer. func (d *Decoder) Uvarint() (uint64, error) { - x, n := binary.Uvarint(d.buf) - if n == 0 { - return 0, io.ErrShortBuffer - } - if n < 0 { - return 0, errUvarintOverflow + x, buf, err := ReadUvarint(d.buf) + if err != nil { + return 0, err } - d.buf = d.buf[n:] + d.buf = buf return x, nil } // Bytes reads a byte slice from the decoder. func (d *Decoder) Bytes() ([]byte, error) { - x, err := d.Uvarint() + b, buf, err := ReadBytes(d.buf) if err != nil { return nil, err } + d.buf = buf + return b, nil +} + +// ReadUvarint reads a variable-size unsigned integer from a byte slice +// and returns a new slice positioned after the integer that was just read. +func ReadUvarint(buf []byte) (uint64, []byte, error) { + x, n := binary.Uvarint(buf) + if n == 0 { + return 0, nil, io.ErrShortBuffer + } + if n < 0 { + return 0, nil, errUvarintOverflow + } + buf = buf[n:] + return x, buf, nil +} + +// ReadBytes reads an unsigned integer from a byte slice and +// returns that amount of bytes along with a new slice positioned after +// the last byte just read. +func ReadBytes(buf []byte) ([]byte, []byte, error) { + x, buf, err := ReadUvarint(buf) + if err != nil { + return nil, nil, err + } // Verify the length of the slice won't overflow an int. if x > uint64(maxInt) { - return nil, errIntOverflow + return nil, nil, errIntOverflow } n := int(x) - if len(d.buf) < n { - return nil, io.ErrShortBuffer + if len(buf) < n { + return nil, nil, io.ErrShortBuffer } - b := d.buf[:n] - d.buf = d.buf[n:] - return b, nil + b := buf[:n] + buf = buf[n:] + return b, buf, nil } diff --git a/src/m3ninx/index/segment/fst/segment.go b/src/m3ninx/index/segment/fst/segment.go index 37091f6965..fd041df720 100644 --- a/src/m3ninx/index/segment/fst/segment.go +++ b/src/m3ninx/index/segment/fst/segment.go @@ -131,9 +131,10 @@ func NewSegment(data SegmentData, opts Options) (Segment, error) { } var ( - docsThirdPartyReader = data.DocsReader - docsDataReader *docs.DataReader - docsIndexReader *docs.IndexReader + docsThirdPartyReader = data.DocsReader + docsDataReader *docs.DataReader + docsEncodedDataReader *docs.EncodedDataReader + docsIndexReader *docs.IndexReader ) if docsThirdPartyReader == nil { docsDataReader = docs.NewDataReader(data.DocsData.Bytes) @@ -142,12 +143,14 @@ func NewSegment(data SegmentData, opts Options) (Segment, error) { return nil, fmt.Errorf("unable to load documents index: %v", err) } } + docsEncodedDataReader = docs.NewEncodedDataReader(data.DocsData.Bytes) s := &fsSegment{ - fieldsFST: fieldsFST, - docsDataReader: docsDataReader, - docsIndexReader: docsIndexReader, - docsThirdPartyReader: docsThirdPartyReader, + fieldsFST: fieldsFST, + docsDataReader: docsDataReader, + docsEncodedDataReader: docsEncodedDataReader, + docsIndexReader: docsIndexReader, + docsThirdPartyReader: docsThirdPartyReader, data: data, opts: opts, @@ -169,15 +172,16 @@ var _ segment.ImmutableSegment = (*fsSegment)(nil) type fsSegment struct { sync.RWMutex - ctx context.Context - closed bool - finalized bool - fieldsFST *vellum.FST - docsDataReader *docs.DataReader - docsIndexReader *docs.IndexReader - docsThirdPartyReader docs.Reader - data SegmentData - opts Options + ctx context.Context + closed bool + finalized bool + fieldsFST *vellum.FST + docsDataReader *docs.DataReader + docsEncodedDataReader *docs.EncodedDataReader + docsIndexReader *docs.IndexReader + docsThirdPartyReader docs.Reader + data SegmentData + opts Options numDocs int64 } @@ -579,7 +583,7 @@ func (r *fsSegment) matchAllNotClosedMaybeFinalizedWithRLock() (postings.Mutable return pl, nil } -func (r *fsSegment) docNotClosedMaybeFinalizedWithRLock(id postings.ID) (doc.Metadata, error) { +func (r *fsSegment) metadataNotClosedMaybeFinalizedWithRLock(id postings.ID) (doc.Metadata, error) { // NB(r): Not closed, but could be finalized (i.e. closed segment reader) // calling match field after this segment is finalized. if r.finalized { @@ -599,6 +603,49 @@ func (r *fsSegment) docNotClosedMaybeFinalizedWithRLock(id postings.ID) (doc.Met return r.docsDataReader.Read(offset) } +func (r *fsSegment) metadataIteratorNotClosedMaybeFinalizedWithRLock( + retriever index.MetadataRetriever, + pl postings.List, +) (doc.MetadataIterator, error) { + // NB(r): Not closed, but could be finalized (i.e. closed segment reader) + // calling match field after this segment is finalized. + if r.finalized { + return nil, errReaderFinalized + } + + return index.NewIDDocIterator(retriever, pl.Iterator()), nil +} + +func (r *fsSegment) docNotClosedMaybeFinalizedWithRLock(id postings.ID) (doc.Document, error) { + // NB(r): Not closed, but could be finalized (i.e. closed segment reader) + // calling match field after this segment is finalized. + if r.finalized { + return doc.Document{}, errReaderFinalized + } + + // If using docs slice reader, return from the in memory slice reader + if r.docsThirdPartyReader != nil { + m, err := r.docsThirdPartyReader.Read(id) + if err != nil { + return doc.Document{}, err + } + + return doc.NewDocumentFromMetadata(m), nil + } + + offset, err := r.docsIndexReader.Read(id) + if err != nil { + return doc.Document{}, err + } + + e, err := r.docsEncodedDataReader.Read(offset) + if err != nil { + return doc.Document{}, err + } + + return doc.NewDocumentFromEncoded(e), nil +} + func (r *fsSegment) docsNotClosedMaybeFinalizedWithRLock( retriever index.DocRetriever, pl postings.List, @@ -609,11 +656,11 @@ func (r *fsSegment) docsNotClosedMaybeFinalizedWithRLock( return nil, errReaderFinalized } - return index.NewIDDocIterator(retriever, pl.Iterator()), nil + return index.NewIterator(retriever, pl.Iterator()), nil } func (r *fsSegment) allDocsNotClosedMaybeFinalizedWithRLock( - retriever index.DocRetriever, + retriever index.MetadataRetriever, ) (index.IDDocIterator, error) { // NB(r): Not closed, but could be finalized (i.e. closed segment reader) // calling match field after this segment is finalized. @@ -896,13 +943,39 @@ func (sr *fsSegmentReader) MatchAll() (postings.MutableList, error) { return pl, err } -func (sr *fsSegmentReader) Doc(id postings.ID) (doc.Metadata, error) { +func (sr *fsSegmentReader) Metadata(id postings.ID) (doc.Metadata, error) { if sr.closed { return doc.Metadata{}, errReaderClosed } // NB(r): We are allowed to call match field after Close called on // the segment but not after it is finalized. sr.fsSegment.RLock() + pl, err := sr.fsSegment.metadataNotClosedMaybeFinalizedWithRLock(id) + sr.fsSegment.RUnlock() + return pl, err +} + +func (sr *fsSegmentReader) MetadataIterator(pl postings.List) (doc.MetadataIterator, error) { + if sr.closed { + return nil, errReaderClosed + } + // NB(r): We are allowed to call match field after Close called on + // the segment but not after it is finalized. + // Also make sure the doc retriever is the reader not the segment so that + // is closed check is not performed and only the is finalized check. + sr.fsSegment.RLock() + iter, err := sr.fsSegment.metadataIteratorNotClosedMaybeFinalizedWithRLock(sr, pl) + sr.fsSegment.RUnlock() + return iter, err +} + +func (sr *fsSegmentReader) Doc(id postings.ID) (doc.Document, error) { + if sr.closed { + return doc.Document{}, errReaderClosed + } + // NB(r): We are allowed to call match field after Close called on + // the segment but not after it is finalized. + sr.fsSegment.RLock() pl, err := sr.fsSegment.docNotClosedMaybeFinalizedWithRLock(id) sr.fsSegment.RUnlock() return pl, err diff --git a/src/m3ninx/index/segment/fst/writer_reader_test.go b/src/m3ninx/index/segment/fst/writer_reader_test.go index 2c785fabcf..eef2fbdf5f 100644 --- a/src/m3ninx/index/segment/fst/writer_reader_test.go +++ b/src/m3ninx/index/segment/fst/writer_reader_test.go @@ -420,9 +420,9 @@ func TestSegmentDocs(t *testing.T) { obsPl, err := obsReader.MatchTerm(f, []byte(term)) require.NoError(t, err) - expDocs, err := expReader.Docs(expPl) + expDocs, err := expReader.MetadataIterator(expPl) require.NoError(t, err) - obsDocs, err := obsReader.Docs(obsPl) + obsDocs, err := obsReader.MetadataIterator(obsPl) require.NoError(t, err) assertDocsEqual(t, expDocs, obsDocs) @@ -529,10 +529,10 @@ func TestSegmentReaderValidUntilClose(t *testing.T) { require.NoError(t, err) assertPostingsList(t, list, []postings.ID{0, 1, 2}) - _, err = reader.Doc(0) + _, err = reader.Metadata(0) require.NoError(t, err) - _, err = reader.Docs(list) + _, err = reader.MetadataIterator(list) require.NoError(t, err) _, err = reader.AllDocs() @@ -543,7 +543,7 @@ func TestSegmentReaderValidUntilClose(t *testing.T) { require.NoError(t, err) list, err = reader.MatchRegexp([]byte("fruit"), re) require.NoError(t, err) - iter, err := reader.Docs(list) + iter, err := reader.MetadataIterator(list) require.NoError(t, err) var docs int for iter.Next() { @@ -589,7 +589,7 @@ func assertSliceOfByteSlicesEqual(t *testing.T, a, b [][]byte) { require.Equal(t, a, b) } -func assertDocsEqual(t *testing.T, a, b doc.Iterator) { +func assertDocsEqual(t *testing.T, a, b doc.MetadataIterator) { aDocs, err := collectDocs(a) require.NoError(t, err) bDocs, err := collectDocs(b) @@ -647,7 +647,7 @@ func assertPostingsList(t *testing.T, l postings.List, exp []postings.ID) { require.Fail(t, msg) } -func collectDocs(iter doc.Iterator) ([]doc.Metadata, error) { +func collectDocs(iter doc.MetadataIterator) ([]doc.Metadata, error) { var docs []doc.Metadata for iter.Next() { docs = append(docs, iter.Current()) diff --git a/src/m3ninx/index/segment/mem/reader.go b/src/m3ninx/index/segment/mem/reader.go index 35993ff0f7..998a184452 100644 --- a/src/m3ninx/index/segment/mem/reader.go +++ b/src/m3ninx/index/segment/mem/reader.go @@ -124,28 +124,45 @@ func (r *reader) MatchAll() (postings.MutableList, error) { return pl, nil } -func (r *reader) Doc(id postings.ID) (doc.Metadata, error) { +func (r *reader) Metadata(id postings.ID) (doc.Metadata, error) { + r.RLock() + defer r.RUnlock() + + return r.getMetadataWithRLock(id) +} + +func (r *reader) MetadataIterator(pl postings.List) (doc.MetadataIterator, error) { r.RLock() defer r.RUnlock() if r.closed { - return doc.Metadata{}, errSegmentReaderClosed + return nil, errSegmentReaderClosed } + boundedIter := newBoundedPostingsIterator(pl.Iterator(), r.limits) + return r.getMetadataIterWithLock(boundedIter), nil +} - if id < r.limits.startInclusive || id >= r.limits.endExclusive { - return doc.Metadata{}, index.ErrDocNotFound +func (r *reader) Doc(id postings.ID) (doc.Document, error) { + r.RLock() + defer r.RUnlock() + + m, err := r.getMetadataWithRLock(id) + if err != nil { + return doc.Document{}, err } - return r.segment.getDoc(id) + return doc.NewDocumentFromMetadata(m), nil } func (r *reader) Docs(pl postings.List) (doc.Iterator, error) { r.RLock() defer r.RUnlock() + if r.closed { return nil, errSegmentReaderClosed } + boundedIter := newBoundedPostingsIterator(pl.Iterator(), r.limits) - return r.getDocIterWithLock(boundedIter), nil + return index.NewIterator(r, boundedIter), nil } func (r *reader) AllDocs() (index.IDDocIterator, error) { @@ -156,13 +173,25 @@ func (r *reader) AllDocs() (index.IDDocIterator, error) { } pi := postings.NewRangeIterator(r.limits.startInclusive, r.limits.endExclusive) - return r.getDocIterWithLock(pi), nil + return r.getMetadataIterWithLock(pi), nil } -func (r *reader) getDocIterWithLock(iter postings.Iterator) index.IDDocIterator { +func (r *reader) getMetadataIterWithLock(iter postings.Iterator) index.IDDocIterator { return index.NewIDDocIterator(r, iter) } +func (r *reader) getMetadataWithRLock(id postings.ID) (doc.Metadata, error) { + if r.closed { + return doc.Metadata{}, errSegmentReaderClosed + } + + if id < r.limits.startInclusive || id >= r.limits.endExclusive { + return doc.Metadata{}, index.ErrDocNotFound + } + + return r.segment.getDoc(id) +} + func (r *reader) Close() error { r.Lock() if r.closed { diff --git a/src/m3ninx/index/segment/mem/reader_test.go b/src/m3ninx/index/segment/mem/reader_test.go index cc3ab7e9b8..f82bfef96f 100644 --- a/src/m3ninx/index/segment/mem/reader_test.go +++ b/src/m3ninx/index/segment/mem/reader_test.go @@ -147,7 +147,7 @@ func TestReaderDocs(t *testing.T) { reader := newReader(segment, readerDocRange{0, maxID}, postings.NewPool(nil, roaring.NewPostingsList)) - iter, err := reader.Docs(postingsList) + iter, err := reader.MetadataIterator(postingsList) require.NoError(t, err) actualDocs := make([]doc.Metadata, 0, len(docs)) diff --git a/src/m3ninx/index/segment/mem/segment_test.go b/src/m3ninx/index/segment/mem/segment_test.go index 3ec0aebb00..7c2379b823 100644 --- a/src/m3ninx/index/segment/mem/segment_test.go +++ b/src/m3ninx/index/segment/mem/segment_test.go @@ -127,7 +127,7 @@ func TestSegmentInsert(t *testing.T) { pl, err := r.MatchTerm(doc.IDReservedFieldName, id) require.NoError(t, err) - iter, err := r.Docs(pl) + iter, err := r.MetadataIterator(pl) require.NoError(t, err) require.True(t, iter.Next()) @@ -186,7 +186,7 @@ func TestSegmentInsertDuplicateID(t *testing.T) { pl, err := r.MatchTerm(doc.IDReservedFieldName, id) require.NoError(t, err) - iter, err := r.Docs(pl) + iter, err := r.MetadataIterator(pl) require.NoError(t, err) require.True(t, iter.Next()) @@ -714,7 +714,7 @@ func TestSegmentReaderMatchExact(t *testing.T) { pl, err := r.MatchTerm([]byte("fruit"), []byte("apple")) require.NoError(t, err) - iter, err := r.Docs(pl) + iter, err := r.MetadataIterator(pl) require.NoError(t, err) actualDocs := make([]doc.Metadata, 0) @@ -845,7 +845,7 @@ func TestSegmentReaderMatchRegex(t *testing.T) { pl, err := r.MatchRegexp(field, index.CompiledRegex{Simple: compiled}) require.NoError(t, err) - iter, err := r.Docs(pl) + iter, err := r.MetadataIterator(pl) require.NoError(t, err) actualDocs := make([]doc.Metadata, 0) @@ -872,7 +872,7 @@ func testDocument(t *testing.T, d doc.Metadata, r index.Reader) { pl, err := r.MatchTerm(name, value) require.NoError(t, err) - iter, err := r.Docs(pl) + iter, err := r.MetadataIterator(pl) require.NoError(t, err) require.True(t, iter.Next()) diff --git a/src/m3ninx/index/segment/segment_mock.go b/src/m3ninx/index/segment/segment_mock.go index 3fc758bb7f..7c86c01359 100644 --- a/src/m3ninx/index/segment/segment_mock.go +++ b/src/m3ninx/index/segment/segment_mock.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/m3db/m3/src/m3ninx/index/segment/types.go -// Copyright (c) 2020 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -181,11 +181,26 @@ func (m *MockReader) EXPECT() *MockReaderMockRecorder { return m.recorder } +// Metadata mocks base method +func (m *MockReader) Metadata(id postings.ID) (doc.Metadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Metadata", id) + ret0, _ := ret[0].(doc.Metadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Metadata indicates an expected call of Metadata +func (mr *MockReaderMockRecorder) Metadata(id interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Metadata", reflect.TypeOf((*MockReader)(nil).Metadata), id) +} + // Doc mocks base method -func (m *MockReader) Doc(id postings.ID) (doc.Metadata, error) { +func (m *MockReader) Doc(id postings.ID) (doc.Document, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Doc", id) - ret0, _ := ret[0].(doc.Metadata) + ret0, _ := ret[0].(doc.Document) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -256,6 +271,21 @@ func (mr *MockReaderMockRecorder) MatchAll() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchAll", reflect.TypeOf((*MockReader)(nil).MatchAll)) } +// MetadataIterator mocks base method +func (m *MockReader) MetadataIterator(pl postings.List) (doc.MetadataIterator, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MetadataIterator", pl) + ret0, _ := ret[0].(doc.MetadataIterator) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MetadataIterator indicates an expected call of MetadataIterator +func (mr *MockReaderMockRecorder) MetadataIterator(pl interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MetadataIterator", reflect.TypeOf((*MockReader)(nil).MetadataIterator), pl) +} + // Docs mocks base method func (m *MockReader) Docs(pl postings.List) (doc.Iterator, error) { m.ctrl.T.Helper() diff --git a/src/m3ninx/index/types.go b/src/m3ninx/index/types.go index 99e4de0c6e..1a005691aa 100644 --- a/src/m3ninx/index/types.go +++ b/src/m3ninx/index/types.go @@ -61,6 +61,7 @@ type Writer interface { // Readable provides a point-in-time accessor to the documents in an index. type Readable interface { + MetadataRetriever DocRetriever // MatchField returns a postings list over all documents which match the given field. @@ -76,8 +77,12 @@ type Readable interface { // MatchAll returns a postings list for all documents known to the Reader. MatchAll() (postings.MutableList, error) - // Docs returns an iterator over the documents whose IDs are in the provided + // MetadataIterator returns an iterator over the metadata whose IDs are in the provided // postings list. + MetadataIterator(pl postings.List) (doc.MetadataIterator, error) + + // Docs returns an iterator over the document whose IDs + // are in the provided postings list. Docs(pl postings.List) (doc.Iterator, error) // AllDocs returns an iterator over the documents known to the Reader. @@ -94,16 +99,22 @@ type CompiledRegex struct { PrefixEnd []byte } +// MetadataRetriever returns the metadata associated with a postings ID. It returns +// ErrDocNotFound if there is no metadata corresponding to the given postings ID. +type MetadataRetriever interface { + Metadata(id postings.ID) (doc.Metadata, error) +} + // DocRetriever returns the document associated with a postings ID. It returns // ErrDocNotFound if there is no document corresponding to the given postings ID. type DocRetriever interface { - Doc(id postings.ID) (doc.Metadata, error) + Doc(id postings.ID) (doc.Document, error) } // IDDocIterator is an extented documents Iterator which can also return the postings // ID of the current document. type IDDocIterator interface { - doc.Iterator + doc.MetadataIterator // PostingsID returns the current document postings ID. PostingsID() postings.ID diff --git a/src/m3ninx/search/executor/executor.go b/src/m3ninx/search/executor/executor.go index e5f606a82f..79eca270fe 100644 --- a/src/m3ninx/search/executor/executor.go +++ b/src/m3ninx/search/executor/executor.go @@ -33,7 +33,7 @@ var ( errExecutorClosed = errors.New("executor is closed") ) -type newIteratorFn func(s search.Searcher, rs index.Readers) (doc.Iterator, error) +type newIteratorFn func(s search.Searcher, rs index.Readers) (doc.MetadataIterator, error) type executor struct { sync.RWMutex @@ -52,7 +52,7 @@ func NewExecutor(rs index.Readers) search.Executor { } } -func (e *executor) Execute(q search.Query) (doc.Iterator, error) { +func (e *executor) Execute(q search.Query) (doc.MetadataIterator, error) { e.RLock() defer e.RUnlock() if e.closed { diff --git a/src/m3ninx/search/executor/executor_test.go b/src/m3ninx/search/executor/executor_test.go index 15d7c90f34..ce8ef85a04 100644 --- a/src/m3ninx/search/executor/executor_test.go +++ b/src/m3ninx/search/executor/executor_test.go @@ -58,7 +58,7 @@ func TestExecutor(t *testing.T) { e := NewExecutor(rs).(*executor) // Override newIteratorFn to return test iterator. - e.newIteratorFn = func(_ search.Searcher, _ index.Readers) (doc.Iterator, error) { + e.newIteratorFn = func(_ search.Searcher, _ index.Readers) (doc.MetadataIterator, error) { return newTestIterator(), nil } diff --git a/src/m3ninx/search/executor/iterator.go b/src/m3ninx/search/executor/iterator.go index da310902e7..c1cc471ef9 100644 --- a/src/m3ninx/search/executor/iterator.go +++ b/src/m3ninx/search/executor/iterator.go @@ -32,13 +32,13 @@ type iterator struct { idx int currDoc doc.Metadata - currIter doc.Iterator + currIter doc.MetadataIterator err error closed bool } -func newIterator(s search.Searcher, rs index.Readers) (doc.Iterator, error) { +func newIterator(s search.Searcher, rs index.Readers) (doc.MetadataIterator, error) { it := &iterator{ searcher: s, readers: rs, @@ -110,7 +110,7 @@ func (it *iterator) Close() error { // nextIter gets the next document iterator by getting the next postings list from // the it's searcher and then getting the documents for that postings list from the // corresponding reader associated with that postings list. -func (it *iterator) nextIter() (doc.Iterator, bool, error) { +func (it *iterator) nextIter() (doc.MetadataIterator, bool, error) { it.idx++ if it.idx >= len(it.readers) { return nil, false, nil @@ -122,7 +122,7 @@ func (it *iterator) nextIter() (doc.Iterator, bool, error) { return nil, false, err } - iter, err := reader.Docs(pl) + iter, err := reader.MetadataIterator(pl) if err != nil { return nil, false, err } diff --git a/src/m3ninx/search/executor/iterator_test.go b/src/m3ninx/search/executor/iterator_test.go index 1e56f4abd7..28cf95f021 100644 --- a/src/m3ninx/search/executor/iterator_test.go +++ b/src/m3ninx/search/executor/iterator_test.go @@ -71,8 +71,8 @@ func TestIterator(t *testing.T) { }, } - firstDocIter := doc.NewMockIterator(mockCtrl) - secondDocIter := doc.NewMockIterator(mockCtrl) + firstDocIter := doc.NewMockMetadataIterator(mockCtrl) + secondDocIter := doc.NewMockMetadataIterator(mockCtrl) gomock.InOrder( firstDocIter.EXPECT().Next().Return(true), firstDocIter.EXPECT().Current().Return(docs[0]), @@ -92,8 +92,8 @@ func TestIterator(t *testing.T) { firstReader := index.NewMockReader(mockCtrl) secondReader := index.NewMockReader(mockCtrl) gomock.InOrder( - firstReader.EXPECT().Docs(firstPL).Return(firstDocIter, nil), - secondReader.EXPECT().Docs(secondPL).Return(secondDocIter, nil), + firstReader.EXPECT().MetadataIterator(firstPL).Return(firstDocIter, nil), + secondReader.EXPECT().MetadataIterator(secondPL).Return(secondDocIter, nil), ) searcher := search.NewMockSearcher(mockCtrl) diff --git a/src/m3ninx/search/proptest/segment_gen.go b/src/m3ninx/search/proptest/segment_gen.go index 41ef29daa7..aa3bf83c4e 100644 --- a/src/m3ninx/search/proptest/segment_gen.go +++ b/src/m3ninx/search/proptest/segment_gen.go @@ -40,7 +40,7 @@ var ( fstOptions = fst.NewOptions() ) -func collectDocs(iter doc.Iterator) ([]doc.Metadata, error) { +func collectDocs(iter doc.MetadataIterator) ([]doc.Metadata, error) { var docs []doc.Metadata for iter.Next() { docs = append(docs, iter.Current()) diff --git a/src/m3ninx/search/proptest/util.go b/src/m3ninx/search/proptest/util.go index 2e2fd9d71c..60dd08e341 100644 --- a/src/m3ninx/search/proptest/util.go +++ b/src/m3ninx/search/proptest/util.go @@ -43,7 +43,7 @@ func newDocumentIteratorMatcher(docs ...doc.Metadata) (*documentIteratorMatcher, } // Matches returns whether the provided iterator matches the collection of provided docs. -func (m *documentIteratorMatcher) Matches(i doc.Iterator) error { +func (m *documentIteratorMatcher) Matches(i doc.MetadataIterator) error { pendingDocIDs := make(map[string]doc.Metadata, len(m.expectedDocs)) for id := range m.expectedDocs { pendingDocIDs[id] = m.expectedDocs[id] diff --git a/src/m3ninx/search/search_mock.go b/src/m3ninx/search/search_mock.go index 3f2e886417..b070619f5f 100644 --- a/src/m3ninx/search/search_mock.go +++ b/src/m3ninx/search/search_mock.go @@ -59,10 +59,10 @@ func (m *MockExecutor) EXPECT() *MockExecutorMockRecorder { } // Execute mocks base method -func (m *MockExecutor) Execute(q Query) (doc.Iterator, error) { +func (m *MockExecutor) Execute(q Query) (doc.MetadataIterator, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Execute", q) - ret0, _ := ret[0].(doc.Iterator) + ret0, _ := ret[0].(doc.MetadataIterator) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/src/m3ninx/search/types.go b/src/m3ninx/search/types.go index d4fc76b4ab..cb21903fda 100644 --- a/src/m3ninx/search/types.go +++ b/src/m3ninx/search/types.go @@ -32,7 +32,7 @@ import ( // Executor is responsible for executing queries over a snapshot. type Executor interface { // Execute executes a query over the Executor's snapshot. - Execute(q Query) (doc.Iterator, error) + Execute(q Query) (doc.MetadataIterator, error) // Close closes the iterator. Close() error