From 2119f5fe2f4323c25b4bb3b31683269e1f9c50c9 Mon Sep 17 00:00:00 2001 From: Albert <26584478+albertteoh@users.noreply.github.com> Date: Sat, 19 Aug 2023 00:35:59 +1000 Subject: [PATCH] Add the enable tracing opt-in flag (#4685) ## Which problem is this PR solving? Resolves #4680 ## Description of the changes - Add an opt-in option `--query.enable-tracing` to enable tracing for the jaeger-query component. - The jaeger all-in-one component does not expose this flag since traces are emitted to port 4317 by default, which all-in-one listens on. ## How was this change tested? ``` # Run jaeger-query component with tracing enabled and verify that the connection errors are appearing in stdout. $ SPAN_STORAGE_TYPE=memory go run -tags ui ./cmd/query/main.go --query.enable-tracing ... {"level":"info","ts":1692363754.9049716,"caller":"grpc/clientconn.go:1301","msg":"[core][Channel #1 SubChannel #2] Subchannel Connectivity change to CONNECTING","system":"grpc","grpc_log":true} {"level":"info","ts":1692363754.9050152,"caller":"grpc/clientconn.go:1414","msg":"[core][Channel #1 SubChannel #2] Subchannel picks a new address \"localhost:4317\" to connect","system":"grpc","grpc_log":true} {"level":"warn","ts":1692363754.9058733,"caller":"grpc/clientconn.go:1476","msg":"[core][Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {Addr: \"localhost:4317\", ServerName: \"localhost:4317\", }. Err: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:4317: connect: connection refused\"","system":"grpc","grpc_log":true} {"level":"info","ts":1692363754.9067123,"caller":"grpc/clientconn.go:1303","msg":"[core][Channel #1 SubChannel #2] Subchannel Connectivity change to TRANSIENT_FAILURE, last error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:4317: connect: connection refused\"","system":"grpc","grpc_log":true} ... # Run jaeger-query component with tracing disabled and verify that the connection errors no longer appear. # Of course, we can't see traces in Jaeger UI because there's nothing to receive the traces. $ SPAN_STORAGE_TYPE=memory go run -tags ui ./cmd/query/main.go # Start an all-in-one instance just as a quick and dirty way to bring up an in-memory jaeger stack to # receive traces from jaeger-query $ make run-all-in-one # Run jaeger-query as a separate component, listening on different ports to all-in-one to avoid port binding collisions. $ SPAN_STORAGE_TYPE=memory go run -tags ui ./cmd/query/main.go --query.enable-tracing --query.grpc-server.host-port :17685 --query.http-server.host-port :17686 --admin.http.host-port :17687 # Open localhost:17686 in a browser and refresh a few times to emit traces to jaeger all-in-one. ``` Confirmed that `jaeger-query` is visible and contains traces: Screenshot 2023-08-18 at 11 45 26 pm ## Checklist - [x] I have read https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md - [x] I have signed all commits ~- [] I have added unit tests for the new functionality~ - [x] I have run lint and test steps successfully - for `jaeger`: `make lint test` - for `jaeger-ui`: `yarn lint` and `yarn test` --------- Signed-off-by: albertteoh --- cmd/all-in-one/main.go | 2 -- cmd/query/app/flags.go | 5 +++++ cmd/query/main.go | 20 ++++++++++++-------- examples/memstore-plugin/main.go | 2 -- pkg/jtracer/jtracer.go | 2 ++ 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/cmd/all-in-one/main.go b/cmd/all-in-one/main.go index 89297fdde2e..34d86f534ce 100644 --- a/cmd/all-in-one/main.go +++ b/cmd/all-in-one/main.go @@ -24,7 +24,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "go.opentelemetry.io/otel" _ "go.uber.org/automaxprocs" "go.uber.org/zap" @@ -105,7 +104,6 @@ by default uses only in-memory database.`, if err != nil { logger.Fatal("Failed to initialize tracer", zap.Error(err)) } - otel.SetTracerProvider(tracer.OTEL) storageFactory.InitFromViper(v, logger) if err := storageFactory.Initialize(metricsFactory, logger); err != nil { diff --git a/cmd/query/app/flags.go b/cmd/query/app/flags.go index faacdb5593b..10c68c5adf5 100644 --- a/cmd/query/app/flags.go +++ b/cmd/query/app/flags.go @@ -47,6 +47,7 @@ const ( queryTokenPropagation = "query.bearer-token-propagation" queryAdditionalHeaders = "query.additional-headers" queryMaxClockSkewAdjust = "query.max-clock-skew-adjustment" + queryEnableTracing = "query.enable-tracing" ) var tlsGRPCFlagsConfig = tlscfg.ServerFlagsConfig{ @@ -85,6 +86,8 @@ type QueryOptions struct { MaxClockSkewAdjust time.Duration // Tenancy configures tenancy for query Tenancy tenancy.Options + // EnableTracing determines whether traces will be emitted by jaeger-query. + EnableTracing bool } // AddFlags adds flags for QueryOptions @@ -98,6 +101,7 @@ func AddFlags(flagSet *flag.FlagSet) { flagSet.String(queryUIConfig, "", "The path to the UI configuration file in JSON format") flagSet.Bool(queryTokenPropagation, false, "Allow propagation of bearer token to be used by storage plugins") flagSet.Duration(queryMaxClockSkewAdjust, 0, "The maximum delta by which span timestamps may be adjusted in the UI due to clock skew; set to 0s to disable clock skew adjustments") + flagSet.Bool(queryEnableTracing, false, "Enables emitting jaeger-query traces") tlsGRPCFlagsConfig.AddFlags(flagSet) tlsHTTPFlagsConfig.AddFlags(flagSet) } @@ -131,6 +135,7 @@ func (qOpts *QueryOptions) InitFromViper(v *viper.Viper, logger *zap.Logger) (*Q qOpts.AdditionalHeaders = headers } qOpts.Tenancy = tenancy.InitFromViper(v) + qOpts.EnableTracing = v.GetBool(queryEnableTracing) return qOpts, nil } diff --git a/cmd/query/main.go b/cmd/query/main.go index 0442dcd06a3..e43d0979346 100644 --- a/cmd/query/main.go +++ b/cmd/query/main.go @@ -23,7 +23,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "go.opentelemetry.io/otel" _ "go.uber.org/automaxprocs" "go.uber.org/zap" @@ -73,15 +72,20 @@ func main() { baseFactory := svc.MetricsFactory.Namespace(metrics.NSOptions{Name: "jaeger"}) metricsFactory := baseFactory.Namespace(metrics.NSOptions{Name: "query"}) version.NewInfoMetrics(metricsFactory) - jtracer, err := jtracer.New("jaeger-query") - if err != nil { - logger.Fatal("Failed to create tracer:", zap.Error(err)) - } - otel.SetTracerProvider(jtracer.OTEL) + queryOpts, err := new(app.QueryOptions).InitFromViper(v, logger) if err != nil { logger.Fatal("Failed to configure query service", zap.Error(err)) } + + jt := jtracer.NoOp() + if queryOpts.EnableTracing { + jt, err = jtracer.New("jaeger-query") + if err != nil { + logger.Fatal("Failed to create tracer", zap.Error(err)) + } + } + // TODO: Need to figure out set enable/disable propagation on storage plugins. v.Set(bearertoken.StoragePropagationKey, queryOpts.BearerTokenPropagation) storageFactory.InitFromViper(v, logger) @@ -108,7 +112,7 @@ func main() { dependencyReader, *queryServiceOptions) tm := tenancy.NewManager(&queryOpts.Tenancy) - server, err := app.NewServer(svc.Logger, queryService, metricsQueryService, queryOpts, tm, jtracer) + server, err := app.NewServer(svc.Logger, queryService, metricsQueryService, queryOpts, tm, jt) if err != nil { logger.Fatal("Failed to create server", zap.Error(err)) } @@ -128,7 +132,7 @@ func main() { if err := storageFactory.Close(); err != nil { logger.Error("Failed to close storage factory", zap.Error(err)) } - if err = jtracer.Close(context.Background()); err != nil { + if err = jt.Close(context.Background()); err != nil { logger.Fatal("Error shutting down tracer provider", zap.Error(err)) } }) diff --git a/examples/memstore-plugin/main.go b/examples/memstore-plugin/main.go index 36a9b31bb07..d8b7dcd4337 100644 --- a/examples/memstore-plugin/main.go +++ b/examples/memstore-plugin/main.go @@ -23,7 +23,6 @@ import ( "github.com/hashicorp/go-plugin" "github.com/spf13/viper" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/otel" googleGRPC "google.golang.org/grpc" "github.com/jaegertracing/jaeger/pkg/jtracer" @@ -62,7 +61,6 @@ func main() { panic(fmt.Errorf("failed to initialize tracer: %w", err)) } defer tracer.Close(context.Background()) - otel.SetTracerProvider(tracer.OTEL) memStorePlugin := grpcMemory.NewStoragePlugin(memory.NewStore(), memory.NewStore()) service := &shared.PluginServices{ diff --git a/pkg/jtracer/jtracer.go b/pkg/jtracer/jtracer.go index eca142b543b..103e208a676 100644 --- a/pkg/jtracer/jtracer.go +++ b/pkg/jtracer/jtracer.go @@ -88,6 +88,8 @@ func initOTEL(ctx context.Context, svc string) (*sdktrace.TracerProvider, error) )) }) + otel.SetTracerProvider(tracerProvider) + return tracerProvider, nil }