Fix integration tests

src/dbnode/integration/write_tagged_quorum_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/m3db/m3/src/cluster/shard"
 	"github.com/m3db/m3/src/dbnode/client"
 	"github.com/m3db/m3/src/dbnode/storage/index"
+	"github.com/m3db/m3/src/dbnode/test"
 	"github.com/m3db/m3/src/dbnode/topology"
 	"github.com/m3db/m3/src/dbnode/x/xio"
 	m3ninxidx "github.com/m3db/m3/src/m3ninx/idx"
@@ -303,9 +304,13 @@ func nodeHasTaggedWrite(t *testing.T, s TestSetup) bool {
 	require.NoError(t, err)
 	results := res.Results
 	require.Equal(t, nsCtx.ID.String(), results.Namespace().String())
-	tags, ok := results.Map().Get(ident.StringID("quorumTest"))
-	idxFound := ok && ident.NewTagIterMatcher(ident.MustNewTagStringsIterator(
-		"foo", "bar", "boo", "baz")).Matches(tags)
+	doc, ok := results.Map().Get(ident.BytesID("quorumTest"))
+	idxFound := false
+	if ok {
+		tags := test.DocumentToTagIter(t, doc)
+		idxFound = ident.NewTagIterMatcher(ident.MustNewTagStringsIterator(
+			"foo", "bar", "boo", "baz")).Matches(tags)
+	}
 
 	if !idxFound {
 		return false

src/dbnode/storage/index/block_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/m3db/m3/src/dbnode/retention"
 	"github.com/m3db/m3/src/dbnode/storage/bootstrap/result"
 	"github.com/m3db/m3/src/dbnode/storage/index/compaction"
+	"github.com/m3db/m3/src/dbnode/test"
 	"github.com/m3db/m3/src/dbnode/tracepoint"
 	"github.com/m3db/m3/src/m3ninx/doc"
 	"github.com/m3db/m3/src/m3ninx/idx"
@@ -584,7 +585,7 @@ func TestBlockMockQueryExecutorExecLimit(t *testing.T) {
 	d, ok := results.Map().Get(testDoc1().ID)
 	require.True(t, ok)
 
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
@@ -676,7 +677,7 @@ func TestBlockMockQuerySeriesLimitNonExhaustive(t *testing.T) {
 	d, ok := results.Map().Get(testDoc1().ID)
 	require.True(t, ok)
 
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
@@ -729,7 +730,7 @@ func TestBlockMockQuerySeriesLimitExhaustive(t *testing.T) {
 	require.Equal(t, 1, rMap.Len())
 	d, ok := rMap.Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
@@ -780,7 +781,7 @@ func TestBlockMockQueryDocsLimitNonExhaustive(t *testing.T) {
 	require.Equal(t, 1, results.Map().Len())
 	d, ok := results.Map().Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
@@ -833,7 +834,7 @@ func TestBlockMockQueryDocsLimitExhaustive(t *testing.T) {
 	require.Equal(t, 1, rMap.Len())
 	d, ok := rMap.Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
@@ -890,7 +891,7 @@ func TestBlockMockQueryMergeResultsMapLimit(t *testing.T) {
 	require.Equal(t, 1, rMap.Len())
 	d, ok := rMap.Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
@@ -948,14 +949,14 @@ func TestBlockMockQueryMergeResultsDupeID(t *testing.T) {
 	require.Equal(t, 2, rMap.Len())
 	d, ok := rMap.Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
 
 	d, ok = rMap.Get(testDoc2().ID)
 	require.True(t, ok)
-	t2 := documentToTagIter(t, d)
+	t2 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches(
 		t2))
@@ -1419,14 +1420,14 @@ func TestBlockE2EInsertQuery(t *testing.T) {
 	rMap := results.Map()
 	d, ok := rMap.Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
 
 	d, ok = rMap.Get(testDoc2().ID)
 	require.True(t, ok)
-	t2 := documentToTagIter(t, d)
+	t2 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches(
 		t2))
@@ -1499,7 +1500,7 @@ func TestBlockE2EInsertQueryLimit(t *testing.T) {
 	d, ok := rMap.Get(testDoc1().ID)
 	if ok {
 		numFound++
-		t1 := documentToTagIter(t, d)
+		t1 := test.DocumentToTagIter(t, d)
 		require.True(t, ident.NewTagIterMatcher(
 			ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 			t1))
@@ -1508,7 +1509,7 @@ func TestBlockE2EInsertQueryLimit(t *testing.T) {
 	d, ok = rMap.Get(testDoc2().ID)
 	if ok {
 		numFound++
-		t2 := documentToTagIter(t, d)
+		t2 := test.DocumentToTagIter(t, d)
 		require.True(t, ident.NewTagIterMatcher(
 			ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches(
 			t2))
@@ -1588,14 +1589,14 @@ func TestBlockE2EInsertAddResultsQuery(t *testing.T) {
 	rMap := results.Map()
 	d, ok := rMap.Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
 
 	d, ok = rMap.Get(testDoc2().ID)
 	require.True(t, ok)
-	t2 := documentToTagIter(t, d)
+	t2 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches(
 		t2))
@@ -1669,14 +1670,14 @@ func TestBlockE2EInsertAddResultsMergeQuery(t *testing.T) {
 	rMap := results.Map()
 	d, ok := results.Map().Get(testDoc1().ID)
 	require.True(t, ok)
-	t1 := documentToTagIter(t, d)
+	t1 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz")).Matches(
 		t1))
 
 	d, ok = rMap.Get(testDoc2().ID)
 	require.True(t, ok)
-	t2 := documentToTagIter(t, d)
+	t2 := test.DocumentToTagIter(t, d)
 	require.True(t, ident.NewTagIterMatcher(
 		ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches(
 		t2))

src/dbnode/storage/index/results_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 
 	idxconvert "github.com/m3db/m3/src/dbnode/storage/index/convert"
+	"github.com/m3db/m3/src/dbnode/test"
 	"github.com/m3db/m3/src/m3ninx/doc"
 	"github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs"
 	"github.com/m3db/m3/src/x/ident"
@@ -128,7 +129,7 @@ func TestResultsFirstInsertWins(t *testing.T) {
 	d, ok := res.Map().Get(d1.ID)
 	require.True(t, ok)
 
-	tags := documentToTagIter(t, d)
+	tags := test.DocumentToTagIter(t, d)
 	require.Equal(t, 0, tags.Remaining())
 
 	d2 := doc.Metadata{
@@ -152,7 +153,7 @@ func TestResultsFirstInsertWins(t *testing.T) {
 	d, ok = res.Map().Get([]byte("abc"))
 	require.True(t, ok)
 
-	tags = documentToTagIter(t, d)
+	tags = test.DocumentToTagIter(t, d)
 	require.Equal(t, 0, tags.Remaining())
 }
 
@@ -169,7 +170,7 @@ func TestResultsInsertContains(t *testing.T) {
 	d, ok := res.Map().Get([]byte("abc"))
 	require.True(t, ok)
 
-	tags := documentToTagIter(t, d)
+	tags := test.DocumentToTagIter(t, d)
 	require.Equal(t, 0, tags.Remaining())
 }
 
@@ -240,7 +241,7 @@ func TestResultsReset(t *testing.T) {
 	d, ok := res.Map().Get([]byte("abc"))
 	require.True(t, ok)
 
-	tags := documentToTagIter(t, d)
+	tags := test.DocumentToTagIter(t, d)
 	require.Equal(t, 0, tags.Remaining())
 
 	res.Reset(nil, QueryResultsOptions{})
@@ -279,7 +280,7 @@ func TestFinalize(t *testing.T) {
 	d, ok := res.Map().Get([]byte("abc"))
 	require.True(t, ok)
 
-	tags := documentToTagIter(t, d)
+	tags := test.DocumentToTagIter(t, d)
 	require.Equal(t, 0, tags.Remaining())
 
 	// Call Finalize() to reset the Results.
@@ -291,11 +292,3 @@ func TestFinalize(t *testing.T) {
 	require.Equal(t, 0, res.Size())
 	require.Equal(t, 0, res.TotalDocsCount())
 }
-
-func documentToTagIter(t *testing.T, doc doc.Document) ident.TagIterator {
-	reader := docs.NewEncodedDocumentReader()
-	m, err := docs.GetFromDocument(doc, reader)
-	require.NoError(t, err)
-
-	return idxconvert.ToSeriesTags(m, idxconvert.Opts{NoClone: true})
-}

src/dbnode/storage/index_query_concurrent_test.go

@@ -29,11 +29,14 @@ import (
 	"testing"
 	"time"
 
+	testutil "github.com/m3db/m3/src/dbnode/test"
+
 	"github.com/m3db/m3/src/dbnode/storage/index"
 	"github.com/m3db/m3/src/dbnode/storage/index/convert"
 	"github.com/m3db/m3/src/m3ninx/doc"
 	"github.com/m3db/m3/src/m3ninx/idx"
 	"github.com/m3db/m3/src/x/context"
+	"github.com/m3db/m3/src/x/ident"
 	xresource "github.com/m3db/m3/src/x/resource"
 	xsync "github.com/m3db/m3/src/x/sync"
 	xtest "github.com/m3db/m3/src/x/test"
@@ -344,23 +347,24 @@ func testNamespaceIndexHighConcurrentQueries(
 
 				// Read the results concurrently too
 				hits := make(map[string]struct{}, results.Results.Size())
+				id := ident.NewReusableBytesID()
 				for _, entry := range results.Results.Map().Iter() {
-					id := entry.Key().String()
-
-					doc, err := convert.FromSeriesIDAndTagIter(entry.Key(), entry.Value())
+					id.Reset(entry.Key())
+					tags := testutil.DocumentToTagIter(t, entry.Value())
+					doc, err := convert.FromSeriesIDAndTagIter(id, tags)
 					require.NoError(t, err)
 					if err != nil {
 						continue // this will fail the test anyway, but don't want to panic
 					}
 
-					expectedDoc, ok := expectedResults[id]
+					expectedDoc, ok := expectedResults[id.String()]
 					require.True(t, ok)
 					if !ok {
 						continue // this will fail the test anyway, but don't want to panic
 					}
 
 					require.Equal(t, expectedDoc, doc)
-					hits[id] = struct{}{}
+					hits[id.String()] = struct{}{}
 				}
 				expectedHits := idsPerBlock * (k + 1)
 				require.Equal(t, expectedHits, len(hits))

src/dbnode/test/util.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2021  Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// Package test is a package for shared test helpers.
+package test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/m3db/m3/src/dbnode/storage/index/convert"
+	"github.com/m3db/m3/src/m3ninx/doc"
+	"github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs"
+	"github.com/m3db/m3/src/x/ident"
+)
+
+// DocumentToTagIter is a help for converting a doc.Document into an
+// ident.TagIterator.
+func DocumentToTagIter(t *testing.T, doc doc.Document) ident.TagIterator {
+	reader := docs.NewEncodedDocumentReader()
+	m, err := docs.GetFromDocument(doc, reader)
+	require.NoError(t, err)
+
+	return convert.ToSeriesTags(m, convert.Opts{NoClone: true})
+}

src/m3ninx/doc/document.go

@@ -221,9 +221,10 @@ type Encoded struct {
 // Document contains either metadata or an encoded metadata
 // but never both.
 type Document struct {
-	metadata Metadata
 	encoded  Encoded
+	metadata Metadata
 
+	hasEncoded  bool
 	hasMetadata bool
 }
 
@@ -234,7 +235,7 @@ func NewDocumentFromMetadata(m Metadata) Document {
 
 // NewDocumentFromEncoded creates a Document from an Encoded.
 func NewDocumentFromEncoded(e Encoded) Document {
-	return Document{encoded: e}
+	return Document{encoded: e, hasEncoded: true}
 }
 
 // Metadata returns the metadata it contains, if it has one. Otherwise returns an empty metadata
@@ -250,7 +251,7 @@ func (d *Document) Metadata() (Metadata, bool) {
 // Encoded returns the encoded metadata it contains, if it has one. Otherwise returns an
 // empty encoded metadata and false.
 func (d *Document) Encoded() (Encoded, bool) {
-	if !d.hasMetadata {
+	if d.hasEncoded {
 		return d.encoded, true
 	}
 

src/m3ninx/search/proptest/issue865_test.go

@@ -87,7 +87,7 @@ func TestAnyDistributionOfDocsDoesNotAffectQuery(t *testing.T) {
 	parameters.Rng = rand.New(rand.NewSource(seed))
 	properties := gopter.NewProperties(parameters)
 
-	docMatcher, err := newDocumentIteratorMatcher(t, doc2)
+	docMatcher, err := newDocumentIteratorMatcher(t, doc.NewDocumentFromMetadata(doc2))
 	require.NoError(t, err)
 	properties.Property("Any distribution of simple documents does not affect query results", prop.ForAll(
 		func(i propTestInput) (bool, error) {