Traces sampler env var (thanos-io#6306)

* Issue#5947 OTEL_TRACES_SAMPLER env var Signed-off-by: shayyxi <[email protected]> * Test correction Signed-off-by: shayyxi <[email protected]> * doc failure correction. parse float argument correction. Signed-off-by: shayyxi <[email protected]> * added the changelog. Signed-off-by: shayyxi <[email protected]> * ran make docs to fix the build failure. Signed-off-by: shayyxi <[email protected]> * corrected the incorrect change in tools.md Signed-off-by: shayyxi <[email protected]> * fixed review comments. Signed-off-by: shayyxi <[email protected]> --------- Signed-off-by: shayyxi <[email protected]> Signed-off-by: Shazi <[email protected]> Co-authored-by: shayyxi <[email protected]>
databricks · Jun 27, 2023 · 4b55f84 · 4b55f84
1 parent 686691f
commit 4b55f84
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#6222](https://github.com/thanos-io/thanos/pull/6222) mixin(Receive): Fix tenant series received charts.
 - [#6218](https://github.com/thanos-io/thanos/pull/6218) mixin(Store): handle ResourceExhausted as a non-server error. As a consequence, this error won't contribute to Store's grpc errors alerts.
 - [#6271](https://github.com/thanos-io/thanos/pull/6271) Receive: Fix segfault in `LabelValues` during head compaction.
+- [#6306](https://github.com/thanos-io/thanos/pull/6306) Tracing: tracing in OTLP utilize the OTEL_TRACES_SAMPLER env variable
 - [#6330](https://github.com/thanos-io/thanos/pull/6330) Store: Fix inconsistent error for series limits.
 
 ### Changed

diff --git a/docs/tracing.md b/docs/tracing.md
@@ -97,6 +97,8 @@ config:
     key_file: ""
     server_name: ""
     insecure_skip_verify: false
+  sampler_type: ""
+  sampler_param: ""
 ```
 
 ### Jaeger

diff --git a/pkg/tracing/otlp/config_yaml.go b/pkg/tracing/otlp/config_yaml.go
@@ -31,6 +31,8 @@ type Config struct {
 	RetryConfig        retryConfig       `yaml:"retry_config"`
 	Headers            map[string]string `yaml:"headers"`
 	TLSConfig          exthttp.TLSConfig `yaml:"tls_config"`
+	SamplerType        string            `yaml:"sampler_type"`
+	SamplerParam       string            `yaml:"sampler_param"`
 }
 
 func traceGRPCOptions(config Config) []otlptracegrpc.Option {

diff --git a/pkg/tracing/otlp/otlp.go b/pkg/tracing/otlp/otlp.go
@@ -5,6 +5,7 @@ package otlp
 
 import (
 	"context"
+	"strconv"
 	"strings"
 
 	"github.com/thanos-io/thanos/pkg/tracing/migration"
@@ -25,6 +26,9 @@ import (
 const (
 	TracingClientGRPC string = "grpc"
 	TracingClientHTTP string = "http"
+	AlwaysSample      string = "alwayssample"
+	NeverSample       string = "neversample"
+	RatioBasedSample  string = "traceidratiobased"
 )
 
 // NewOTELTracer returns an OTLP exporter based tracer.
@@ -59,12 +63,16 @@ func NewTracerProvider(ctx context.Context, logger log.Logger, conf []byte) (*tr
 	}
 
 	processor := tracesdk.NewBatchSpanProcessor(exporter)
-	tp := newTraceProvider(ctx, processor, logger, config.ServiceName)
+	sampler, err := getSampler(config)
+	if err != nil {
+		logger.Log(err)
+	}
+	tp := newTraceProvider(ctx, processor, logger, config.ServiceName, sampler)
 
 	return tp, nil
 }
 
-func newTraceProvider(ctx context.Context, processor tracesdk.SpanProcessor, logger log.Logger, serviceName string) *tracesdk.TracerProvider {
+func newTraceProvider(ctx context.Context, processor tracesdk.SpanProcessor, logger log.Logger, serviceName string, sampler tracesdk.Sampler) *tracesdk.TracerProvider {
 	resource, err := resource.New(
 		ctx,
 		resource.WithAttributes(semconv.ServiceNameKey.String(serviceName)),
@@ -73,8 +81,6 @@ func newTraceProvider(ctx context.Context, processor tracesdk.SpanProcessor, log
 		level.Warn(logger).Log("msg", "jaeger: detecting resources for tracing provider failed", "err", err)
 	}
 
-	sampler := tracesdk.ParentBased(tracesdk.TraceIDRatioBased(1.0))
-
 	tp := tracesdk.NewTracerProvider(
 		tracesdk.WithSpanProcessor(processor),
 		tracesdk.WithResource(resource),
@@ -86,3 +92,20 @@ func newTraceProvider(ctx context.Context, processor tracesdk.SpanProcessor, log
 	)
 	return tp
 }
+
+func getSampler(config Config) (tracesdk.Sampler, error) {
+	switch strings.ToLower(config.SamplerType) {
+	case AlwaysSample:
+		return tracesdk.ParentBased(tracesdk.AlwaysSample()), nil
+	case NeverSample:
+		return tracesdk.ParentBased(tracesdk.NeverSample()), nil
+	case RatioBasedSample:
+		arg, err := strconv.ParseFloat(config.SamplerParam, 64)
+		if err != nil {
+			return tracesdk.ParentBased(tracesdk.TraceIDRatioBased(1.0)), err
+		}
+		return tracesdk.ParentBased(tracesdk.TraceIDRatioBased(arg)), nil
+	}
+
+	return tracesdk.ParentBased(tracesdk.TraceIDRatioBased(1.0)), nil
+}
diff --git a/pkg/tracing/otlp/otlp_test.go b/pkg/tracing/otlp/otlp_test.go
@@ -24,7 +24,8 @@ func TestContextTracing_ClientEnablesTracing(t *testing.T) {
 		context.Background(),
 		tracesdk.NewSimpleSpanProcessor(exp),
 		log.NewNopLogger(),
-		"thanos")
+		"thanos",
+		tracesdk.AlwaysSample())
 	tracer, _ := migration.Bridge(tracerOtel, log.NewNopLogger())
 	clientRoot, _ := tracing.StartSpan(tracing.ContextWithTracer(context.Background(), tracer), "a")