Support Jaeger tracer thanos-io#1030 (thanos-io#1147)

* Add jaeger tracing feature.

* Remove comments

* Remove comments

* Refactoring tracing

* Implementing jaeger logger.

* Add jaeger force tracing header.

* Use debugName for tracing service-name

* RemoveFactory config

* Formatting fix; Use io.Closer intead func() error

* Rename gcloud => stackdriver

* Refactoring google tracing testing.

* Delete comments.

* Rename gcloud flags => stackdriver.

* Update tracing docs.

* Fix ..traceType to ..tracerType

* Refactoring NoopTracer; Comments for exported functions.

* Remove noop tracer. Fix docs.

* Remove noop tracer. Fix docs.

* Config tracing same as objstore.

* Configure jaeger tracing from YAML. Some small fixes.

* Fix errcheck

* Add X-Thanos-Trace-Id HTTP header for simplified search traces

* Cleanup

* make docs

* Add store addr to tracing tags.

* Tracing refactoring.

* Fix noop tracing closer.

* Add few tracing spans.

* Pass prometheus registry to jaeger.

* go.mod

* Refactoring

* Resolve go mod conflicts

* Remove comments.

* PR refactoring for review.

* Format files.

cmd/thanos/flags.go

@@ -113,3 +113,24 @@ func regCommonObjStoreFlags(cmd *kingpin.CmdClause, suffix string, required bool
 		content: bucketConf,
 	}
 }
+
+
+func regCommonTracingFlags(app *kingpin.Application) *pathOrContent {
+	fileFlagName := fmt.Sprintf("tracing.config-file")
+	contentFlagName := fmt.Sprintf("tracing.config")
+
+	help := fmt.Sprintf("Path to YAML file that contains tracing configuration.")
+	tracingConfFile := app.Flag(fileFlagName, help).PlaceHolder("<tracing.config-yaml-path>").String()
+
+	help = fmt.Sprintf("Alternative to '%s' flag. Tracing configuration in YAML.", fileFlagName)
+	tracingConf := app.Flag(contentFlagName, help).PlaceHolder("<tracing.config-yaml>").String()
+
+	return &pathOrContent{
+		fileFlagName:    fileFlagName,
+		contentFlagName: contentFlagName,
+		required:        false,
+
+		path:    tracingConfFile,
+		content: tracingConf,
+	}
+}

cmd/thanos/main.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math"
 	"net"
@@ -18,17 +19,17 @@ import (
 	"syscall"
 
 	gmetrics "github.com/armon/go-metrics"
-
 	gprom "github.com/armon/go-metrics/prometheus"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
-	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
+	"github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
-	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/improbable-eng/thanos/pkg/runutil"
 	"github.com/improbable-eng/thanos/pkg/tracing"
+	"github.com/improbable-eng/thanos/pkg/tracing/client"
 	"github.com/oklog/run"
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -37,7 +38,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/status"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"gopkg.in/alecthomas/kingpin.v2"
 )
 
 const (
@@ -65,10 +66,7 @@ func main() {
 	logFormat := app.Flag("log.format", "Log format to use.").
 		Default(logFormatLogfmt).Enum(logFormatLogfmt, logFormatJson)
 
-	gcloudTraceProject := app.Flag("gcloudtrace.project", "GCP project to send Google Cloud Trace tracings to. If empty, tracing will be disabled.").
-		String()
-	gcloudTraceSampleFactor := app.Flag("gcloudtrace.sample-factor", "How often we send traces (1/<sample-factor>). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details.").
-		Default("1").Uint64()
+	tracingConfig := regCommonTracingFlags(app)
 
 	cmds := map[string]setupFunc{}
 	registerSidecar(cmds, app, "sidecar")
@@ -144,8 +142,20 @@ func main() {
 	{
 		ctx := context.Background()
 
-		var closeFn func() error
-		tracer, closeFn = tracing.NewOptionalGCloudTracer(ctx, logger, *gcloudTraceProject, *gcloudTraceSampleFactor, *debugName)
+		var closer io.Closer
+		var confContentYaml []byte
+		confContentYaml, err = tracingConfig.Content()
+
+		if len(confContentYaml) == 0 {
+			level.Info(logger).Log("msg", "Tracing will be disabled")
+			tracer = client.NoopTracer()
+		} else {
+			tracer, closer, err = client.NewTracer(ctx, logger, metrics, confContentYaml)
+			if err != nil {
+				fmt.Fprintln(os.Stderr, errors.Wrapf(err, "tracing failed"))
+				os.Exit(1)
+			}
+		}
 
 		// This is bad, but Prometheus does not support any other tracer injections than just global one.
 		// TODO(bplotka): Work with basictracer to handle gracefully tracker mismatches, and also with Prometheus to allow
@@ -157,8 +167,10 @@ func main() {
 			<-ctx.Done()
 			return ctx.Err()
 		}, func(error) {
-			if err := closeFn(); err != nil {
-				level.Warn(logger).Log("msg", "closing tracer failed", "err", err)
+			if closer != nil {
+				if err := closer.Close(); err != nil {
+					level.Warn(logger).Log("msg", "closing tracer failed", "err", err)
+				}
 			}
 			cancel()
 		})

cmd/thanos/sidecar.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"math"
 	"net"
-	"net/http"
 	"net/url"
 	"sync"
 	"time"
@@ -21,13 +20,13 @@ import (
 	"github.com/improbable-eng/thanos/pkg/store"
 	"github.com/improbable-eng/thanos/pkg/store/storepb"
 	"github.com/oklog/run"
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/tsdb/labels"
 	"google.golang.org/grpc"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"gopkg.in/alecthomas/kingpin.v2"
 )
 
 func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name string) {
@@ -204,10 +203,8 @@ func runSidecar(
 		}
 		logger := log.With(logger, "component", component.Sidecar.String())
 
-		var client http.Client
-
 		promStore, err := store.NewPrometheusStore(
-			logger, &client, promURL, component.Sidecar, m.Labels, m.Timestamps)
+			logger, nil, promURL, component.Sidecar, m.Labels, m.Timestamps)
 		if err != nil {
 			return errors.Wrap(err, "create Prometheus store")
 		}

docs/components/bucket.md

@@ -42,14 +42,12 @@ Flags:
       --version            Show application version.
       --log.level=info     Log filtering level.
       --log.format=logfmt  Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                           GCP project to send Google Cloud Trace tracings to.
-                           If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                           How often we send traces (1/<sample-factor>). If 0 no
-                           trace will be sent periodically, unless forced by
-                           baggage item. See `pkg/tracing/tracing.go` for
-                           details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                           Path to YAML file that contains tracing
+                           configuration.
+      --tracing.config=<tracing.config-yaml>
+                           Alternative to 'tracing.config-file' flag. Tracing
+                           configuration in YAML.
       --objstore.config-file=<bucket.config-yaml-path>
                            Path to YAML file that contains object store
                            configuration.
@@ -92,14 +90,12 @@ Flags:
       --version            Show application version.
       --log.level=info     Log filtering level.
       --log.format=logfmt  Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                           GCP project to send Google Cloud Trace tracings to.
-                           If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                           How often we send traces (1/<sample-factor>). If 0 no
-                           trace will be sent periodically, unless forced by
-                           baggage item. See `pkg/tracing/tracing.go` for
-                           details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                           Path to YAML file that contains tracing
+                           configuration.
+      --tracing.config=<tracing.config-yaml>
+                           Alternative to 'tracing.config-file' flag. Tracing
+                           configuration in YAML.
       --objstore.config-file=<bucket.config-yaml-path>
                            Path to YAML file that contains object store
                            configuration.
@@ -149,14 +145,12 @@ Flags:
       --version            Show application version.
       --log.level=info     Log filtering level.
       --log.format=logfmt  Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                           GCP project to send Google Cloud Trace tracings to.
-                           If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                           How often we send traces (1/<sample-factor>). If 0 no
-                           trace will be sent periodically, unless forced by
-                           baggage item. See `pkg/tracing/tracing.go` for
-                           details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                           Path to YAML file that contains tracing
+                           configuration.
+      --tracing.config=<tracing.config-yaml>
+                           Alternative to 'tracing.config-file' flag. Tracing
+                           configuration in YAML.
       --objstore.config-file=<bucket.config-yaml-path>
                            Path to YAML file that contains object store
                            configuration.
@@ -190,14 +184,12 @@ Flags:
       --version              Show application version.
       --log.level=info       Log filtering level.
       --log.format=logfmt    Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                             GCP project to send Google Cloud Trace tracings to.
-                             If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                             How often we send traces (1/<sample-factor>). If 0
-                             no trace will be sent periodically, unless forced
-                             by baggage item. See `pkg/tracing/tracing.go` for
-                             details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                             Path to YAML file that contains tracing
+                             configuration.
+      --tracing.config=<tracing.config-yaml>
+                             Alternative to 'tracing.config-file' flag. Tracing
+                             configuration in YAML.
       --objstore.config-file=<bucket.config-yaml-path>
                              Path to YAML file that contains object store
                              configuration.

docs/components/compact.md

@@ -40,14 +40,12 @@ Flags:
       --version                Show application version.
       --log.level=info         Log filtering level.
       --log.format=logfmt      Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                               GCP project to send Google Cloud Trace tracings
-                               to. If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                               How often we send traces (1/<sample-factor>). If
-                               0 no trace will be sent periodically, unless
-                               forced by baggage item. See
-                               `pkg/tracing/tracing.go` for details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                               Path to YAML file that contains tracing
+                               configuration.
+      --tracing.config=<tracing.config-yaml>
+                               Alternative to 'tracing.config-file' flag.
+                               Tracing configuration in YAML.
       --http-address="0.0.0.0:10902"
                                Listen host:port for HTTP endpoints.
       --data-dir="./data"      Data directory in which to cache blocks and

docs/components/query.md

@@ -174,14 +174,12 @@ Flags:
       --version                  Show application version.
       --log.level=info           Log filtering level.
       --log.format=logfmt        Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                                 GCP project to send Google Cloud Trace tracings
-                                 to. If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                                 How often we send traces (1/<sample-factor>).
-                                 If 0 no trace will be sent periodically, unless
-                                 forced by baggage item. See
-                                 `pkg/tracing/tracing.go` for details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                                 Path to YAML file that contains tracing
+                                 configuration.
+      --tracing.config=<tracing.config-yaml>
+                                 Alternative to 'tracing.config-file' flag.
+                                 Tracing configuration in YAML.
       --http-address="0.0.0.0:10902"
                                  Listen host:port for HTTP endpoints.
       --grpc-address="0.0.0.0:10901"

docs/components/rule.md

@@ -157,14 +157,12 @@ Flags:
       --version                  Show application version.
       --log.level=info           Log filtering level.
       --log.format=logfmt        Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                                 GCP project to send Google Cloud Trace tracings
-                                 to. If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                                 How often we send traces (1/<sample-factor>).
-                                 If 0 no trace will be sent periodically, unless
-                                 forced by baggage item. See
-                                 `pkg/tracing/tracing.go` for details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                                 Path to YAML file that contains tracing
+                                 configuration.
+      --tracing.config=<tracing.config-yaml>
+                                 Alternative to 'tracing.config-file' flag.
+                                 Tracing configuration in YAML.
       --http-address="0.0.0.0:10902"
                                  Listen host:port for HTTP endpoints.
       --grpc-address="0.0.0.0:10901"

docs/components/sidecar.md

@@ -56,14 +56,12 @@ Flags:
       --version                  Show application version.
       --log.level=info           Log filtering level.
       --log.format=logfmt        Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                                 GCP project to send Google Cloud Trace tracings
-                                 to. If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                                 How often we send traces (1/<sample-factor>).
-                                 If 0 no trace will be sent periodically, unless
-                                 forced by baggage item. See
-                                 `pkg/tracing/tracing.go` for details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                                 Path to YAML file that contains tracing
+                                 configuration.
+      --tracing.config=<tracing.config-yaml>
+                                 Alternative to 'tracing.config-file' flag.
+                                 Tracing configuration in YAML.
       --http-address="0.0.0.0:10902"
                                  Listen host:port for HTTP endpoints.
       --grpc-address="0.0.0.0:10901"

docs/components/store.md

@@ -41,14 +41,12 @@ Flags:
       --version                  Show application version.
       --log.level=info           Log filtering level.
       --log.format=logfmt        Log format to use.
-      --gcloudtrace.project=GCLOUDTRACE.PROJECT
-                                 GCP project to send Google Cloud Trace tracings
-                                 to. If empty, tracing will be disabled.
-      --gcloudtrace.sample-factor=1
-                                 How often we send traces (1/<sample-factor>).
-                                 If 0 no trace will be sent periodically, unless
-                                 forced by baggage item. See
-                                 `pkg/tracing/tracing.go` for details.
+      --tracing.config-file=<tracing.config-yaml-path>
+                                 Path to YAML file that contains tracing
+                                 configuration.
+      --tracing.config=<tracing.config-yaml>
+                                 Alternative to 'tracing.config-file' flag.
+                                 Tracing configuration in YAML.
       --http-address="0.0.0.0:10902"
                                  Listen host:port for HTTP endpoints.
       --grpc-address="0.0.0.0:10901"

go.mod

@@ -29,12 +29,16 @@ require (
 	github.com/olekukonko/tablewriter v0.0.1
 	github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7
 	github.com/opentracing/basictracer-go v1.0.0
-	github.com/opentracing/opentracing-go v1.0.2
+	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/prometheus/common v0.4.0
 	github.com/prometheus/prometheus v2.9.2+incompatible
 	github.com/prometheus/tsdb v0.8.0
+	github.com/uber-go/atomic v1.4.0 // indirect
+	github.com/uber/jaeger-client-go v2.16.0+incompatible
+	github.com/uber/jaeger-lib v2.0.0+incompatible
+	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
 	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
 	golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6