Add m3ninx query proptests

glide.yaml

@@ -220,4 +220,4 @@ testImport:
   version: b433bbd6d743c1854040b39062a3916ed5f78fe8
 
 - package: github.com/leanovate/gopter
-  version: f778776473e0ef7764e1434dd01a61cc1ec574b4
+  version: f0356731348c8fffa27bab27c37ec8be5b0662c8

src/m3ninx/index/segment/fst/encoding/docs/data_test.go

@@ -25,7 +25,7 @@ import (
 	"testing"
 
 	"github.com/m3db/m3/src/m3ninx/doc"
-	"github.com/m3db/m3/src/m3ninx/index/util"
+	"github.com/m3db/m3/src/m3ninx/util"
 
 	"github.com/stretchr/testify/require"
 )
@@ -76,7 +76,7 @@ func TestStoredFieldsData(t *testing.T) {
 		},
 		{
 			name: "node exporter metrics",
-			docs: util.MustReadDocs("../../../../util/testdata/node_exporter.json", 2000),
+			docs: util.MustReadDocs("../../../../../util/testdata/node_exporter.json", 2000),
 		},
 	}
 

src/m3ninx/index/segment/fst/writer_reader_test.go

@@ -31,8 +31,8 @@ import (
 	"github.com/m3db/m3/src/m3ninx/index"
 	sgmt "github.com/m3db/m3/src/m3ninx/index/segment"
 	"github.com/m3db/m3/src/m3ninx/index/segment/mem"
-	"github.com/m3db/m3/src/m3ninx/index/util"
 	"github.com/m3db/m3/src/m3ninx/postings"
+	"github.com/m3db/m3/src/m3ninx/util"
 
 	"github.com/stretchr/testify/require"
 )
@@ -79,7 +79,7 @@ var (
 			},
 		},
 	}
-	lotsTestDocuments = util.MustReadDocs("../../util/testdata/node_exporter.json", 2000)
+	lotsTestDocuments = util.MustReadDocs("../../../util/testdata/node_exporter.json", 2000)
 
 	testDocuments = []struct {
 		name string

src/m3ninx/index/segment/mem/options.go

@@ -21,9 +21,9 @@
 package mem
 
 import (
-	"github.com/m3db/m3/src/m3ninx/index/util"
 	"github.com/m3db/m3/src/m3ninx/postings"
 	"github.com/m3db/m3/src/m3ninx/postings/roaring"
+	"github.com/m3db/m3/src/m3ninx/util"
 	"github.com/m3db/m3/src/m3ninx/x/bytes"
 
 	"github.com/m3db/m3x/instrument"

src/m3ninx/index/segment/mem/segment.go

@@ -28,8 +28,8 @@ import (
 	"github.com/m3db/m3/src/m3ninx/doc"
 	"github.com/m3db/m3/src/m3ninx/index"
 	sgmt "github.com/m3db/m3/src/m3ninx/index/segment"
-	"github.com/m3db/m3/src/m3ninx/index/util"
 	"github.com/m3db/m3/src/m3ninx/postings"
+	"github.com/m3db/m3/src/m3ninx/util"
 )
 
 var (

src/m3ninx/index/segment/mem/segment_bench_test.go

@@ -25,7 +25,7 @@ import (
 	"testing"
 
 	"github.com/m3db/m3/src/m3ninx/doc"
-	"github.com/m3db/m3/src/m3ninx/index/util"
+	"github.com/m3db/m3/src/m3ninx/util"
 )
 
 var (
@@ -53,7 +53,7 @@ func BenchmarkSegment(b *testing.B) {
 		},
 	}
 
-	docs, err := util.ReadDocs("../../util/testdata/node_exporter.json", 2000)
+	docs, err := util.ReadDocs("../../../util/testdata/node_exporter.json", 2000)
 	if err != nil {
 		b.Fatalf("unable to read documents for benchmarks: %v", err)
 	}

src/m3ninx/index/segment/mem/terms_dict_bench_test.go

@@ -25,8 +25,8 @@ import (
 	"testing"
 
 	"github.com/m3db/m3/src/m3ninx/doc"
-	"github.com/m3db/m3/src/m3ninx/index/util"
 	"github.com/m3db/m3/src/m3ninx/postings"
+	"github.com/m3db/m3/src/m3ninx/util"
 )
 
 var (
@@ -54,7 +54,7 @@ func BenchmarkTermsDict(b *testing.B) {
 		},
 	}
 
-	docs, err := util.ReadDocs("../../util/testdata/node_exporter.json", 2000)
+	docs, err := util.ReadDocs("../../../util/testdata/node_exporter.json", 2000)
 	if err != nil {
 		b.Fatalf("unable to read documents for benchmarks: %v", err)
 	}

src/m3ninx/search/proptest/issue865_test.go

@@ -23,36 +23,25 @@
 package proptest
 
 import (
-	"fmt"
 	"math/rand"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/m3db/m3/src/m3ninx/doc"
 	"github.com/m3db/m3/src/m3ninx/index"
-	"github.com/m3db/m3/src/m3ninx/index/segment"
-	"github.com/m3db/m3/src/m3ninx/index/segment/fst"
-	"github.com/m3db/m3/src/m3ninx/index/segment/mem"
-	"github.com/m3db/m3/src/m3ninx/postings"
 	"github.com/m3db/m3/src/m3ninx/search"
 	"github.com/m3db/m3/src/m3ninx/search/executor"
 	"github.com/m3db/m3/src/m3ninx/search/query"
 
 	"github.com/leanovate/gopter"
-	"github.com/leanovate/gopter/gen"
 	"github.com/leanovate/gopter/prop"
 	"github.com/stretchr/testify/require"
 )
 
 // NB(prateek): this test simulates the issues described in issue: https://github.com/m3db/m3/issues/865
-// tl;dr - the searcher code assumes the input readers had disjoint doc ID ranges; it caused issues when that
-// was not true.
 
 var (
-	memOptions = mem.NewOptions()
-	fstOptions = fst.NewOptions()
-
 	doc1 = doc.Document{
 		ID: []byte("__name__=node_cpu_seconds_total,cpu=1,instance=m3db-node01:9100,job=node-exporter,mode=system,"),
 		Fields: []doc.Field{
@@ -98,6 +87,8 @@ func TestAnyDistributionOfDocsDoesNotAffectQuery(t *testing.T) {
 	parameters.Rng = rand.New(rand.NewSource(seed))
 	properties := gopter.NewProperties(parameters)
 
+	docMatcher, err := newDocumentIteratorMatcher(doc2)
+	require.NoError(t, err)
 	properties.Property("Any distribution of simple documents does not affect query results", prop.ForAll(
 		func(i propTestInput) (bool, error) {
 			segments := i.generate(t, simpleTestDocs)
@@ -121,21 +112,7 @@ func TestAnyDistributionOfDocsDoesNotAffectQuery(t *testing.T) {
 				return false, err
 			}
 
-			if !d.Next() {
-				return false, fmt.Errorf("unable to find any documents")
-			}
-
-			curr := d.Current()
-			if !curr.Equal(doc2) {
-				return false, fmt.Errorf("returned document [%+v] did not match exepcted document [%+v]",
-					curr, doc2)
-			}
-
-			if d.Next() {
-				return false, fmt.Errorf("found too many documents")
-			}
-
-			if err := d.Err(); err != nil {
+			if err := docMatcher.Matches(d); err != nil {
 				return false, err
 			}
 
@@ -149,103 +126,3 @@ func TestAnyDistributionOfDocsDoesNotAffectQuery(t *testing.T) {
 		t.Errorf("failed with initial seed: %d", seed)
 	}
 }
-
-func (i propTestInput) generate(t *testing.T, docs []doc.Document) []segment.Segment {
-	var result []segment.Segment
-	for j := 0; j < len(i.segments); j++ {
-		initialOffset := postings.ID(i.segments[j].initialDocIDOffset)
-		s, err := mem.NewSegment(initialOffset, memOptions)
-		require.NoError(t, err)
-		for k := 0; k < len(i.docIds[j]); k++ {
-			idx := i.docIds[j][k]
-			_, err = s.Insert(docs[idx])
-			require.NoError(t, err)
-		}
-
-		if i.segments[j].simpleSegment {
-			result = append(result, s)
-			continue
-		}
-
-		result = append(result, fst.ToTestSegment(t, s, fstOptions))
-	}
-	return result
-}
-
-type propTestInput struct {
-	segments []generatedSegment
-	docIds   [][]int
-}
-
-func genPropTestInput(numDocs int) gopter.Gen {
-	return func(genParams *gopter.GenParameters) *gopter.GenResult {
-		numSegmentsRes, ok := gen.IntRange(1, numDocs)(genParams).Retrieve()
-		if !ok {
-			panic("unable to generate segments")
-		}
-		numSegments := numSegmentsRes.(int)
-
-		docIds := make([]int, 0, numDocs)
-		for i := 0; i < numDocs; i++ {
-			docIds = append(docIds, i)
-		}
-
-		randomIds := randomDocIds(docIds)
-		randomIds.shuffle(genParams.Rng)
-
-		genSegments := make([]generatedSegment, 0, numSegments)
-		partitionedDocs := make([][]int, 0, numSegments)
-		for i := 0; i < numSegments; i++ {
-			partitionedDocs = append(partitionedDocs, []int{})
-			segRes, ok := genSegment()(genParams).Retrieve()
-			if !ok {
-				panic("unable to generate segments")
-			}
-			genSegments = append(genSegments, segRes.(generatedSegment))
-		}
-
-		for i := 0; i < numDocs; i++ {
-			idx := i % numSegments
-			partitionedDocs[idx] = append(partitionedDocs[idx], randomIds[i])
-		}
-
-		result := propTestInput{
-			segments: genSegments,
-			docIds:   partitionedDocs,
-		}
-		if len(genSegments) != len(partitionedDocs) {
-			panic(fmt.Errorf("unequal lengths of segments and docs: %+v", result))
-		}
-
-		return gopter.NewGenResult(result, gopter.NoShrinker)
-	}
-}
-
-func genSegment() gopter.Gen {
-	return gopter.CombineGens(
-		gen.Bool(),         // simple segment
-		gen.IntRange(1, 5), // initial doc id offset
-	).Map(func(val interface{}) generatedSegment {
-		inputs := val.([]interface{})
-		return generatedSegment{
-			simpleSegment:      inputs[0].(bool),
-			initialDocIDOffset: inputs[1].(int),
-		}
-	})
-}
-
-type generatedSegment struct {
-	simpleSegment      bool
-	initialDocIDOffset int
-}
-
-type randomDocIds []int
-
-func (d randomDocIds) shuffle(rng *rand.Rand) {
-	// Start from the last element and swap one by one.
-	// NB: We don't need to run for the first element that's why i > 0
-	for i := len(d) - 1; i > 0; i-- {
-		j := rng.Intn(i)
-		d[i], d[j] = d[j], d[i]
-	}
-}