Skip to content

Commit

Permalink
Add full integration test for graphite find endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
arnikola committed Mar 22, 2019
1 parent 4a54dc5 commit 98c0729
Show file tree
Hide file tree
Showing 8 changed files with 458 additions and 20 deletions.
2 changes: 1 addition & 1 deletion scripts/development/m3_stack/start_m3.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ if [[ "$FORCE_BUILD" = true ]] ; then
fi

echo "Bringing up nodes in the background with docker compose, remember to run ./stop.sh when done"
docker-compose -f docker-compose.yml up --build $DOCKER_ARGS m3coordinator01
docker-compose -f docker-compose.yml up $DOCKER_ARGS m3coordinator01
docker-compose -f docker-compose.yml up $DOCKER_ARGS m3db_seed
docker-compose -f docker-compose.yml up $DOCKER_ARGS prometheus01
docker-compose -f docker-compose.yml up $DOCKER_ARGS grafana
Expand Down
57 changes: 48 additions & 9 deletions src/query/api/v1/handler/graphite/find.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package graphite

import (
"context"
"errors"
"net/http"

"github.com/m3db/m3/src/query/api/v1/handler"
Expand Down Expand Up @@ -56,34 +57,72 @@ func NewFindHandler(
}
}

func mergeTags(
noChildren *storage.CompleteTagsResult,
withChildren *storage.CompleteTagsResult,
) (map[string]bool, error) {
// sanity check the case.
if noChildren.CompleteNameOnly {
return nil, errors.New("tags result with no children is completing name only")
}

if withChildren.CompleteNameOnly {
return nil, errors.New("tag result with children is completing name only")
}

mapLength := len(noChildren.CompletedTags) + len(withChildren.CompletedTags)
tagMap := make(map[string]bool, mapLength)

for _, tag := range noChildren.CompletedTags {
for _, value := range tag.Values {
tagMap[string(value)] = false
}
}

// NB: fine to overwrite any tags which were present in the `noChildren` map
for _, tag := range withChildren.CompletedTags {
for _, value := range tag.Values {
tagMap[string(value)] = true
}
}

return tagMap, nil
}

func (h *grahiteFindHandler) ServeHTTP(
w http.ResponseWriter,
r *http.Request,
) {
ctx := context.WithValue(r.Context(), handler.HeaderKey, r.Header)
logger := logging.WithContext(ctx)
w.Header().Set("Content-Type", "application/json")
query, raw, rErr := parseFindParamsToQuery(r)
noChildrenQuery, childrenQuery, raw, rErr := parseFindParamsToQueries(r)
if rErr != nil {
xhttp.Error(w, rErr.Inner(), rErr.Code())
return
}

opts := storage.NewFetchOptions()
result, err := h.storage.CompleteTags(ctx, query, opts)
noChildrenResult, err := h.storage.CompleteTags(ctx, noChildrenQuery, opts)
if err != nil {
logger.Error("unable to complete tags", zap.Error(err))
xhttp.Error(w, err, http.StatusBadRequest)
return
}

seenMap := make(map[string]bool, len(result.CompletedTags))
for _, tags := range result.CompletedTags {
for _, value := range tags.Values {
// FIXME: (arnikola) Figure out how to add children; may need to run find
// query twice, once with an additional wildcard matcher on the end.
seenMap[string(value)] = true
}
childrenResult, err := h.storage.CompleteTags(ctx, childrenQuery, opts)
if err != nil {
logger.Error("unable to complete tags", zap.Error(err))
xhttp.Error(w, err, http.StatusBadRequest)
return
}

// NB: merge results from both queries to specify which series have children
seenMap, err := mergeTags(noChildrenResult, childrenResult)
if err != nil {
logger.Error("unable to complete tags", zap.Error(err))
xhttp.Error(w, err, http.StatusBadRequest)
return
}

prefix := graphite.DropLastMetricPart(raw)
Expand Down
33 changes: 27 additions & 6 deletions src/query/api/v1/handler/graphite/find_parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,32 +27,53 @@ import (

"github.com/m3db/m3/src/query/errors"
graphiteStorage "github.com/m3db/m3/src/query/graphite/storage"
"github.com/m3db/m3/src/query/models"
"github.com/m3db/m3/src/query/storage"
"github.com/m3db/m3/src/query/util/json"
"github.com/m3db/m3/src/x/net/http"
)

func parseFindParamsToQuery(r *http.Request) (
func parseFindParamsToQueries(r *http.Request) (
*storage.CompleteTagsQuery,
*storage.CompleteTagsQuery,
string,
*xhttp.ParseError,
) {
values := r.URL.Query()
query := values.Get("query")
if query == "" {
return nil, "", xhttp.NewParseError(errors.ErrNoQueryFound, http.StatusBadRequest)
return nil, nil, "", xhttp.NewParseError(errors.ErrNoQueryFound, http.StatusBadRequest)
}

matchers, err := graphiteStorage.TranslateQueryToMatchers(query)
matchers, err := graphiteStorage.TranslateQueryToMatchersWithTerminator(query)
if err != nil {
return nil, "", xhttp.NewParseError(fmt.Errorf("invalid 'query': %s", query),
return nil, nil, "", xhttp.NewParseError(fmt.Errorf("invalid 'query': %s", query),
http.StatusBadRequest)
}

return &storage.CompleteTagsQuery{
// NB: Filter will always be the second last term in the matchers, and the
// matchers should always have a length of at least 2 (term + terminator)
// so this is a sanity check and unexpected in actual execution.
filter := [][]byte{matchers[len(matchers)-2].Name}

noChildren := &storage.CompleteTagsQuery{
CompleteNameOnly: false,
FilterNameTags: filter,
TagMatchers: matchers,
}, query, nil
}

clonedMatchers := make([]models.Matcher, len(matchers))
copy(clonedMatchers, matchers)
// NB: change terminator from `MatchNotRegexp` to `MatchRegexp` to ensure
// segments with children are matched.
clonedMatchers[len(clonedMatchers)-1].Type = models.MatchRegexp
withChildren := &storage.CompleteTagsQuery{
CompleteNameOnly: false,
FilterNameTags: filter,
TagMatchers: clonedMatchers,
}

return noChildren, withChildren, query, nil
}

func findResultsJSON(
Expand Down
217 changes: 217 additions & 0 deletions src/query/api/v1/handler/graphite/find_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package graphite

import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/url"
"sort"
"strings"
"testing"

"github.com/m3db/m3/src/query/models"
"github.com/m3db/m3/src/query/storage"
"github.com/m3db/m3/src/query/util/logging"

"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
)

type completeTagQueryMatcher struct {
matchers []models.Matcher
}

func (m *completeTagQueryMatcher) String() string { return "complete tag query" }
func (m *completeTagQueryMatcher) Matches(x interface{}) bool {
q, ok := x.(*storage.CompleteTagsQuery)
if !ok {
return false
}

if q.CompleteNameOnly {
return false
}

if len(q.FilterNameTags) != 1 {
return false
}

// both queries should filter on __g1__
if !bytes.Equal(q.FilterNameTags[0], []byte("__g1__")) {
return false
}

if len(q.TagMatchers) != len(m.matchers) {
return false
}

for i, qMatcher := range q.TagMatchers {
if !bytes.Equal(qMatcher.Name, m.matchers[i].Name) {
return false
}
if !bytes.Equal(qMatcher.Value, m.matchers[i].Value) {
return false
}
if qMatcher.Type != m.matchers[i].Type {
return false
}
}

return true
}

var _ gomock.Matcher = &completeTagQueryMatcher{}

func b(s string) []byte { return []byte(s) }
func bs(ss ...string) [][]byte {
bb := make([][]byte, len(ss))
for i, s := range ss {
bb[i] = b(s)
}

return bb
}

func setupStorage(ctrl *gomock.Controller) storage.Storage {
store := storage.NewMockStorage(ctrl)
// set up no children case
noChildrenMatcher := &completeTagQueryMatcher{
matchers: []models.Matcher{
{Type: models.MatchRegexp, Name: b("__g0__"), Value: b("foo")},
{Type: models.MatchRegexp, Name: b("__g1__"), Value: b("b.*")},
{Type: models.MatchNotRegexp, Name: b("__g2__"), Value: b(".*")},
},
}

noChildrenResult := &storage.CompleteTagsResult{
CompleteNameOnly: false,
CompletedTags: []storage.CompletedTag{
{Name: b("__g1__"), Values: bs("bug", "bar", "baz")},
},
}

store.EXPECT().CompleteTags(gomock.Any(), noChildrenMatcher, gomock.Any()).
Return(noChildrenResult, nil)

// set up children case
childrenMatcher := &completeTagQueryMatcher{
matchers: []models.Matcher{
{Type: models.MatchRegexp, Name: b("__g0__"), Value: b("foo")},
{Type: models.MatchRegexp, Name: b("__g1__"), Value: b("b.*")},
{Type: models.MatchRegexp, Name: b("__g2__"), Value: b(".*")},
},
}

childrenResult := &storage.CompleteTagsResult{
CompleteNameOnly: false,
CompletedTags: []storage.CompletedTag{
{Name: b("__g1__"), Values: bs("baz", "bix", "bug")},
},
}

store.EXPECT().CompleteTags(gomock.Any(), childrenMatcher, gomock.Any()).
Return(childrenResult, nil)

return store
}

type writer struct {
results []string
}

var _ http.ResponseWriter = &writer{}

func (w *writer) WriteHeader(_ int) {}
func (w *writer) Header() http.Header { return make(http.Header) }
func (w *writer) Write(b []byte) (int, error) {
if w.results == nil {
w.results = make([]string, 0, 10)
}

w.results = append(w.results, string(b))
return len(b), nil
}

type result struct {
ID string `json:"id"`
Text string `json:"text"`
Leaf int `json:"leaf"`
Expandable int `json:"expandable"`
AllowChildren int `json:"allowChildren"`
}

type results []result

func (r results) Len() int { return len(r) }
func (r results) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r results) Less(i, j int) bool {
return strings.Compare(r[i].ID, r[j].ID) == -1
}

func TestFind(t *testing.T) {
logging.InitWithCores(nil)

ctrl := gomock.NewController(t)
defer ctrl.Finish()

// setup storage and handler
store := setupStorage(ctrl)
handler := NewFindHandler(store)

// execute the query
w := &writer{}
req := &http.Request{
URL: &url.URL{
RawQuery: "query=foo.b*",
},
}

handler.ServeHTTP(w, req)

// convert results to comparable format
require.Equal(t, 1, len(w.results))
r := make(results, 0)
decoder := json.NewDecoder(bytes.NewBufferString((w.results[0])))
require.NoError(t, decoder.Decode(&r))
sort.Sort(r)

makeNoChildrenResult := func(t string) result {
return result{ID: fmt.Sprintf("foo.%s", t), Text: t, Leaf: 1,
Expandable: 0, AllowChildren: 0}
}

makeWithChildrenResult := func(t string) result {
return result{ID: fmt.Sprintf("foo.%s", t), Text: t, Leaf: 0,
Expandable: 1, AllowChildren: 1}
}

expected := results{
makeNoChildrenResult("bar"),
makeWithChildrenResult("baz"),
makeWithChildrenResult("bix"),
makeWithChildrenResult("bug"),
}

require.Equal(t, expected, r)
}
1 change: 1 addition & 0 deletions src/query/generated/mocks/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

// mockgen rules for generating mocks for exported interfaces (reflection mode).
//go:generate sh -c "mockgen -package=downsample $PACKAGE/src/cmd/services/m3coordinator/downsample Downsampler,MetricsAppender,SamplesAppender | genclean -pkg $PACKAGE/src/cmd/services/m3coordinator/downsample -out $GOPATH/src/$PACKAGE/src/cmd/services/m3coordinator/downsample/downsample_mock.go"
//go:generate sh -c "mockgen -package=storage -destination=$GOPATH/src/$PACKAGE/src/query/storage/storage_mock.go $PACKAGE/src/query/storage Storage"
//go:generate sh -c "mockgen -package=block -destination=$GOPATH/src/$PACKAGE/src/query/block/block_mock.go $PACKAGE/src/query/block Block,StepIter,SeriesIter,Builder,Step"
//go:generate sh -c "mockgen -package=ingest -destination=$GOPATH/src/$PACKAGE/src/cmd/services/m3coordinator/ingest/write_mock.go $PACKAGE/src/cmd/services/m3coordinator/ingest DownsamplerAndWriter"
//go:generate sh -c "mockgen -package=transform -destination=$GOPATH/src/$PACKAGE/src/query/executor/transform/types_mock.go $PACKAGE/src/query/executor/transform OpNode"
Expand Down
Loading

0 comments on commit 98c0729

Please sign in to comment.