From e50a400043d5559778ed4a2bb8a44ea7903bcff5 Mon Sep 17 00:00:00 2001 From: arnikola Date: Thu, 2 May 2019 13:10:54 -0400 Subject: [PATCH] [query] add list tags endpoint (#1565) --- src/query/api/v1/handler/prometheus/common.go | 29 ++++ .../v1/handler/prometheus/native/list_tags.go | 89 +++++++++++ .../prometheus/native/list_tags_test.go | 145 ++++++++++++++++++ src/query/api/v1/httpd/handler.go | 7 + src/query/errors/handler.go | 2 + src/query/models/matcher.go | 2 + src/query/models/types.go | 1 + src/query/storage/index.go | 3 + src/query/storage/index_test.go | 9 ++ src/query/tsdb/remote/codecs.go | 2 + 10 files changed, 289 insertions(+) create mode 100644 src/query/api/v1/handler/prometheus/native/list_tags.go create mode 100644 src/query/api/v1/handler/prometheus/native/list_tags_test.go diff --git a/src/query/api/v1/handler/prometheus/common.go b/src/query/api/v1/handler/prometheus/common.go index 696bf02cba..5ec181e575 100644 --- a/src/query/api/v1/handler/prometheus/common.go +++ b/src/query/api/v1/handler/prometheus/common.go @@ -304,6 +304,35 @@ func renderDefaultTagCompletionResultsJSON( return jw.Close() } +// RenderListTagResultsJSON renders list tag results to json format. +func RenderListTagResultsJSON( + w io.Writer, + result *storage.CompleteTagsResult, +) error { + if !result.CompleteNameOnly { + return errors.ErrWithNames + } + + jw := json.NewWriter(w) + jw.BeginObject() + + jw.BeginObjectField("status") + jw.WriteString("success") + + jw.BeginObjectField("data") + jw.BeginArray() + + for _, t := range result.CompletedTags { + jw.WriteString(string(t.Name)) + } + + jw.EndArray() + + jw.EndObject() + + return jw.Close() +} + // RenderTagCompletionResultsJSON renders tag completion results to json format. func RenderTagCompletionResultsJSON( w io.Writer, diff --git a/src/query/api/v1/handler/prometheus/native/list_tags.go b/src/query/api/v1/handler/prometheus/native/list_tags.go new file mode 100644 index 0000000000..c014c73b92 --- /dev/null +++ b/src/query/api/v1/handler/prometheus/native/list_tags.go @@ -0,0 +1,89 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package native + +import ( + "context" + "net/http" + "time" + + "github.com/m3db/m3/src/query/api/v1/handler" + "github.com/m3db/m3/src/query/api/v1/handler/prometheus" + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/query/util/logging" + "github.com/m3db/m3/src/x/net/http" + + "go.uber.org/zap" +) + +const ( + // ListTagsURL is the url for listing tags. + ListTagsURL = handler.RoutePrefixV1 + "/labels" +) + +var ( + // ListTagsHTTPMethods are the HTTP methods used with this resource. + ListTagsHTTPMethods = []string{http.MethodGet, http.MethodPost} +) + +// ListTagsHandler represents a handler for list tags endpoint. +type ListTagsHandler struct { + storage storage.Storage +} + +// NewListTagsHandler returns a new instance of handler. +func NewListTagsHandler( + storage storage.Storage, +) http.Handler { + return &ListTagsHandler{ + storage: storage, + } +} + +func (h *ListTagsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := context.WithValue(r.Context(), handler.HeaderKey, r.Header) + logger := logging.WithContext(ctx) + w.Header().Set("Content-Type", "application/json") + + query := &storage.CompleteTagsQuery{ + CompleteNameOnly: true, + TagMatchers: models.Matchers{{Type: models.MatchAll}}, + + // NB: necessarily spans entire possible query range. + Start: time.Time{}, + End: time.Now(), + } + + opts := storage.NewFetchOptions() + result, err := h.storage.CompleteTags(ctx, query, opts) + if err != nil { + logger.Error("unable to complete tags", zap.Error(err)) + xhttp.Error(w, err, http.StatusBadRequest) + return + } + + if err = prometheus.RenderListTagResultsJSON(w, result); err != nil { + logger.Error("unable to render results", zap.Error(err)) + xhttp.Error(w, err, http.StatusBadRequest) + return + } +} diff --git a/src/query/api/v1/handler/prometheus/native/list_tags_test.go b/src/query/api/v1/handler/prometheus/native/list_tags_test.go new file mode 100644 index 0000000000..f28f02bf1f --- /dev/null +++ b/src/query/api/v1/handler/prometheus/native/list_tags_test.go @@ -0,0 +1,145 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package native + +import ( + "errors" + "fmt" + "io/ioutil" + "math" + "net/http/httptest" + "testing" + "time" + + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/query/util/logging" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +type listTagsMatcher struct{} + +func (m *listTagsMatcher) String() string { return "list tags query" } +func (m *listTagsMatcher) Matches(x interface{}) bool { + q, ok := x.(*storage.CompleteTagsQuery) + if !ok { + return false + } + + if !q.Start.Equal(time.Time{}) { + return false + } + + // NB: end time for the query should be roughly `Now` + diff := q.End.Sub(time.Now()) + absDiff := time.Duration(math.Abs(float64(diff))) + if absDiff > time.Second { + return false + } + + if !q.CompleteNameOnly { + return false + } + + if len(q.FilterNameTags) != 0 { + return false + } + + if len(q.TagMatchers) != 1 { + return false + } + + return models.MatchAll == q.TagMatchers[0].Type +} + +var _ gomock.Matcher = &listTagsMatcher{} + +func b(s string) []byte { return []byte(s) } + +func TestListTags(t *testing.T) { + logging.InitWithCores(nil) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // setup storage and handler + store := storage.NewMockStorage(ctrl) + storeResult := &storage.CompleteTagsResult{ + CompleteNameOnly: true, + CompletedTags: []storage.CompletedTag{ + {Name: b("bar")}, + {Name: b("baz")}, + {Name: b("foo")}, + }, + } + + handler := NewListTagsHandler(store) + for _, method := range []string{"GET", "POST"} { + store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). + Return(storeResult, nil) + + req := httptest.NewRequest(method, "/labels", nil) + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + body := w.Result().Body + defer body.Close() + + r, err := ioutil.ReadAll(body) + require.NoError(t, err) + + ex := `{"status":"success","data":["bar","baz","foo"]}` + require.Equal(t, ex, string(r)) + } +} + +func TestListErrorTags(t *testing.T) { + logging.InitWithCores(nil) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // setup storage and handler + store := storage.NewMockStorage(ctrl) + handler := NewListTagsHandler(store) + + for _, method := range []string{"GET", "POST"} { + store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). + Return(nil, errors.New("err")) + + req := httptest.NewRequest(method, "/labels", nil) + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + body := w.Result().Body + defer body.Close() + + r, err := ioutil.ReadAll(body) + require.NoError(t, err) + + ex := `{"error":"err"}` + // NB: error handler adds a newline to the output. + ex = fmt.Sprintf("%s\n", ex) + require.Equal(t, ex, string(r)) + } +} diff --git a/src/query/api/v1/httpd/handler.go b/src/query/api/v1/httpd/handler.go index 7e62b67f98..53552a5244 100644 --- a/src/query/api/v1/httpd/handler.go +++ b/src/query/api/v1/httpd/handler.go @@ -217,6 +217,13 @@ func (h *Handler) RegisterRoutes() error { wrapped(remote.NewTagValuesHandler(h.storage)).ServeHTTP, ).Methods(remote.TagValuesHTTPMethod) + // List tag endpoints + for _, method := range native.ListTagsHTTPMethods { + h.router.HandleFunc(native.ListTagsURL, + wrapped(native.NewListTagsHandler(h.storage)).ServeHTTP, + ).Methods(method) + } + // Series match endpoints h.router.HandleFunc(remote.PromSeriesMatchURL, wrapped(remote.NewPromSeriesMatchHandler(h.storage, h.tagOptions)).ServeHTTP, diff --git a/src/query/errors/handler.go b/src/query/errors/handler.go index b942eb4d41..e066472a3c 100644 --- a/src/query/errors/handler.go +++ b/src/query/errors/handler.go @@ -40,6 +40,8 @@ var ( ErrInvalidMatchers = errors.New("invalid matchers") // ErrNamesOnly is returned when label values results are name only ErrNamesOnly = errors.New("can not render label values; result has label names only") + // ErrWithNames is returned when label values results are name only + ErrWithNames = errors.New("can not render label list; result has label names and values") // ErrMultipleResults is returned when there are multiple label values results ErrMultipleResults = errors.New("can not render label values; multiple results detected") ) diff --git a/src/query/models/matcher.go b/src/query/models/matcher.go index de53ec586d..350f0ee948 100644 --- a/src/query/models/matcher.go +++ b/src/query/models/matcher.go @@ -38,6 +38,8 @@ func (m MatchType) String() string { return "=~" case MatchNotRegexp: return "!~" + case MatchAll: + return "*" default: return "unknown match type" } diff --git a/src/query/models/types.go b/src/query/models/types.go index dfff733a5f..f211d6d80a 100644 --- a/src/query/models/types.go +++ b/src/query/models/types.go @@ -120,6 +120,7 @@ const ( MatchNotEqual MatchRegexp MatchNotRegexp + MatchAll ) // Matcher models the matching of a label. diff --git a/src/query/storage/index.go b/src/query/storage/index.go index de293185fa..769fc77e4f 100644 --- a/src/query/storage/index.go +++ b/src/query/storage/index.go @@ -192,6 +192,9 @@ func matcherToQuery(matcher models.Matcher) (idx.Query, error) { } return query, nil + case models.MatchAll: + return idx.NewAllQuery(), nil + default: return idx.Query{}, fmt.Errorf("unsupported query type: %v", matcher) } diff --git a/src/query/storage/index_test.go b/src/query/storage/index_test.go index 95a4bb9e93..4b40a7a054 100644 --- a/src/query/storage/index_test.go +++ b/src/query/storage/index_test.go @@ -158,6 +158,15 @@ func TestFetchQueryToM3Query(t *testing.T) { expected: "all()", matchers: models.Matchers{}, }, + { + name: "all matchers", + expected: "all()", + matchers: models.Matchers{ + { + Type: models.MatchAll, + }, + }, + }, } for _, test := range tests { diff --git a/src/query/tsdb/remote/codecs.go b/src/query/tsdb/remote/codecs.go index 380260f8b2..d2d18bb38a 100644 --- a/src/query/tsdb/remote/codecs.go +++ b/src/query/tsdb/remote/codecs.go @@ -192,6 +192,8 @@ func encodeMatcherTypeToProto(t models.MatchType) (rpc.MatcherType, error) { return rpc.MatcherType_REGEXP, nil case models.MatchNotRegexp: return rpc.MatcherType_NOTREGEXP, nil + case models.MatchAll: + return rpc.MatcherType_EXISTS, nil default: return rpc.MatcherType_EQUAL, fmt.Errorf("Unknown matcher type for proto encoding") }