diff --git a/CHANGELOG.md b/CHANGELOG.md index ce256defea..62c152b1d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re ### Added +- [#4299](https://github.com/thanos-io/thanos/pull/4299) Tracing: Add tracing to exemplar APIs. - [#4117](https://github.com/thanos-io/thanos/pull/4117) Mixin: new alert ThanosReceiveTrafficBelowThreshold to flag if the ingestion average of the last hour dips below 50% of the ingestion average for the last 12 hours. - [#4107](https://github.com/thanos-io/thanos/pull/4107) Store: `LabelNames` and `LabelValues` now support label matchers. - [#3940](https://github.com/thanos-io/thanos/pull/3940) Sidecar: Added matchers support to `LabelValues` diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index f0aec67ff6..b74dcf864a 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -791,6 +791,15 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo } return func(r *http.Request) (interface{}, []error, *api.ApiError) { + span, ctx := tracing.StartSpan(r.Context(), "exemplar_query_request") + defer span.Finish() + + var ( + data []*exemplarspb.ExemplarData + warnings storage.Warnings + err error + ) + start, err := cortexutil.ParseTime(r.FormValue("start")) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} @@ -806,11 +815,15 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo Query: r.FormValue("query"), PartialResponseStrategy: ps, } - exemplarsData, warnings, err := client.Exemplars(r.Context(), req) + + tracing.DoInSpan(ctx, "retrieve_exemplars", func(ctx context.Context) { + data, warnings, err = client.Exemplars(ctx, req) + }) + if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(err, "retrieving exemplars")} } - return exemplarsData, warnings, nil + return data, warnings, nil } } diff --git a/pkg/exemplars/exemplars.go b/pkg/exemplars/exemplars.go index a5145a9b0f..fab73d4814 100644 --- a/pkg/exemplars/exemplars.go +++ b/pkg/exemplars/exemplars.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" "github.com/thanos-io/thanos/pkg/store/labelpb" + "github.com/thanos-io/thanos/pkg/tracing" ) var _ UnaryClient = &GRPCClient{} @@ -73,6 +74,9 @@ func NewGRPCClientWithDedup(es exemplarspb.ExemplarsServer, replicaLabels []stri } func (rr *GRPCClient) Exemplars(ctx context.Context, req *exemplarspb.ExemplarsRequest) ([]*exemplarspb.ExemplarData, storage.Warnings, error) { + span, ctx := tracing.StartSpan(ctx, "exemplar_grpc_request") + defer span.Finish() + resp := &exemplarsServer{ctx: ctx} if err := rr.proxy.Exemplars(req, resp); err != nil { diff --git a/pkg/exemplars/prometheus.go b/pkg/exemplars/prometheus.go index 2ab3b5f648..c681fd2b16 100644 --- a/pkg/exemplars/prometheus.go +++ b/pkg/exemplars/prometheus.go @@ -4,12 +4,14 @@ package exemplars import ( + "context" "net/url" "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/store/labelpb" + "github.com/thanos-io/thanos/pkg/tracing" ) // Prometheus implements exemplarspb.Exemplars gRPC that allows to fetch exemplars from Prometheus. @@ -41,7 +43,12 @@ func (p *Prometheus) Exemplars(r *exemplarspb.ExemplarsRequest, s exemplarspb.Ex for _, e := range exemplars { // Make sure the returned series labels are sorted. e.SetSeriesLabels(labelpb.ExtendSortedLabels(e.SeriesLabels.PromLabels(), extLset)) - if err := s.Send(&exemplarspb.ExemplarsResponse{Result: &exemplarspb.ExemplarsResponse_Data{Data: e}}); err != nil { + + var err error + tracing.DoInSpan(s.Context(), "send_exemplars_response", func(_ context.Context) { + err = s.Send(&exemplarspb.ExemplarsResponse{Result: &exemplarspb.ExemplarsResponse_Data{Data: e}}) + }) + if err != nil { return err } } diff --git a/pkg/exemplars/proxy.go b/pkg/exemplars/proxy.go index 0386aa816c..d8e2df178e 100644 --- a/pkg/exemplars/proxy.go +++ b/pkg/exemplars/proxy.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/tracing" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -52,6 +53,9 @@ type exemplarsStream struct { } func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exemplars_ExemplarsServer) error { + span, ctx := tracing.StartSpan(srv.Context(), "proxy_exemplars") + defer span.Finish() + expr, err := parser.ParseExpr(req.Query) if err != nil { return err @@ -72,7 +76,7 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe } var ( - g, gctx = errgroup.WithContext(srv.Context()) + g, gctx = errgroup.WithContext(ctx) respChan = make(chan *exemplarspb.ExemplarData, 10) exemplars []*exemplarspb.ExemplarData ) @@ -147,7 +151,10 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe } for _, e := range exemplars { - if err := srv.Send(exemplarspb.NewExemplarsResponse(e)); err != nil { + tracing.DoInSpan(srv.Context(), "send_exemplars_response", func(_ context.Context) { + err = srv.Send(exemplarspb.NewExemplarsResponse(e)) + }) + if err != nil { return status.Error(codes.Unknown, errors.Wrap(err, "send exemplars response").Error()) } }