TracerProvider allows calling Tracer() while it's shutting down

CHANGELOG.md

@@ -33,6 +33,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 ### Fixed
 
 - `TracerProvider` consistently doesn't allow to register a `SpanProcessor` after shutdown. (#3845)
+- `TracerProvider` allows calling `Tracer()` while it's shutting down. It used to deadlock. (#3924)
 
 ### Removed
 

sdk/trace/provider.go

@@ -76,7 +76,8 @@ type TracerProvider struct {
 	mu             sync.Mutex
 	namedTracer    map[instrumentation.Scope]*tracer
 	spanProcessors atomic.Value
-	isShutdown     bool
+
+	isShutdown atomic.Bool
 
 	// These fields are not protected by the lock mu. They are assumed to be
 	// immutable after creation of the TracerProvider.
@@ -136,10 +137,11 @@ func NewTracerProvider(opts ...TracerProviderOption) *TracerProvider {
 //
 // This method is safe to be called concurrently.
 func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer {
+	// This check happens before the mutex is acquired to avoid deadlocking if Tracer() is called from within Shutdown().
+	if p.isShutdown.Load() {
+		return trace.NewNoopTracerProvider().Tracer(name, opts...)
+	}
 	c := trace.NewTracerConfig(opts...)
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
 	if name == "" {
 		name = defaultTracerName
 	}
@@ -148,14 +150,24 @@ func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.T
 		Version:   c.InstrumentationVersion(),
 		SchemaURL: c.SchemaURL(),
 	}
+
+	p.mu.Lock()
+	// Must check the flag after acquiring the mutex to avoid returning a valid tracer if Shutdown() ran
+	// after the first check above but before we acquired the mutex.
+	if p.isShutdown.Load() {
+		return trace.NewNoopTracerProvider().Tracer(name, opts...)
+	}
 	t, ok := p.namedTracer[is]
 	if !ok {
 		t = &tracer{
 			provider:             p,
 			instrumentationScope: is,
 		}
 		p.namedTracer[is] = t
-		global.Info("Tracer created", "name", name, "version", c.InstrumentationVersion(), "schemaURL", c.SchemaURL())
+	}
+	p.mu.Unlock()
+	if !ok {
+		global.Info("Tracer created", "name", name, "version", is.Version, "schemaURL", is.SchemaURL)
 	}
 	return t
 }
@@ -164,7 +176,7 @@ func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.T
 func (p *TracerProvider) RegisterSpanProcessor(sp SpanProcessor) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	if p.isShutdown {
+	if p.isShutdown.Load() {
 		return
 	}
 	newSPS := spanProcessorStates{}
@@ -177,7 +189,7 @@ func (p *TracerProvider) RegisterSpanProcessor(sp SpanProcessor) {
 func (p *TracerProvider) UnregisterSpanProcessor(sp SpanProcessor) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	if p.isShutdown {
+	if p.isShutdown.Load() {
 		return
 	}
 	old := p.spanProcessors.Load().(spanProcessorStates)
@@ -239,10 +251,10 @@ func (p *TracerProvider) ForceFlush(ctx context.Context) error {
 func (p *TracerProvider) Shutdown(ctx context.Context) error {
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	if p.isShutdown {
+	if p.isShutdown.Load() {
 		return nil
 	}
-	p.isShutdown = true
+	p.isShutdown.Store(true)
 	spss := p.spanProcessors.Load().(spanProcessorStates)
 
 	var retErr error

sdk/trace/provider_test.go

@@ -46,11 +46,38 @@ func (t *basicSpanProcessor) ForceFlush(context.Context) error {
 	return nil
 }
 
+type shutdownSpanProcessor struct {
+	shutdown func(context.Context) error
+}
+
+func (t *shutdownSpanProcessor) Shutdown(ctx context.Context) error {
+	return t.shutdown(ctx)
+}
+
+func (t *shutdownSpanProcessor) OnStart(context.Context, ReadWriteSpan) {}
+func (t *shutdownSpanProcessor) OnEnd(ReadOnlySpan)                     {}
+func (t *shutdownSpanProcessor) ForceFlush(context.Context) error {
+	return nil
+}
+
+func TestShutdownCallsTracerMethod(t *testing.T) {
+	stp := NewTracerProvider()
+	sp := &shutdownSpanProcessor{
+		shutdown: func(ctx context.Context) error {
+			_ = stp.Tracer("abc") // must not deadlock
+			return nil
+		},
+	}
+	stp.RegisterSpanProcessor(sp)
+	assert.NoError(t, stp.Shutdown(context.Background()))
+	assert.True(t, stp.isShutdown.Load())
+}
+
 func TestForceFlushAndShutdownTraceProviderWithoutProcessor(t *testing.T) {
 	stp := NewTracerProvider()
 	assert.NoError(t, stp.ForceFlush(context.Background()))
 	assert.NoError(t, stp.Shutdown(context.Background()))
-	assert.True(t, stp.isShutdown)
+	assert.True(t, stp.isShutdown.Load())
 }
 
 func TestShutdownTraceProvider(t *testing.T) {
@@ -61,7 +88,7 @@ func TestShutdownTraceProvider(t *testing.T) {
 	assert.NoError(t, stp.ForceFlush(context.Background()))
 	assert.True(t, sp.flushed, "error ForceFlush basicSpanProcessor")
 	assert.NoError(t, stp.Shutdown(context.Background()))
-	assert.True(t, stp.isShutdown)
+	assert.True(t, stp.isShutdown.Load())
 	assert.True(t, sp.closed, "error Shutdown basicSpanProcessor")
 }
 
@@ -76,7 +103,7 @@ func TestFailedProcessorShutdown(t *testing.T) {
 	err := stp.Shutdown(context.Background())
 	assert.Error(t, err)
 	assert.Equal(t, err, spErr)
-	assert.True(t, stp.isShutdown)
+	assert.True(t, stp.isShutdown.Load())
 }
 
 func TestFailedProcessorsShutdown(t *testing.T) {
@@ -97,7 +124,7 @@ func TestFailedProcessorsShutdown(t *testing.T) {
 	assert.EqualError(t, err, "basic span processor shutdown failure1; basic span processor shutdown failure2")
 	assert.True(t, sp1.closed)
 	assert.True(t, sp2.closed)
-	assert.True(t, stp.isShutdown)
+	assert.True(t, stp.isShutdown.Load())
 }
 
 func TestFailedProcessorShutdownInUnregister(t *testing.T) {
@@ -114,7 +141,7 @@ func TestFailedProcessorShutdownInUnregister(t *testing.T) {
 
 	err := stp.Shutdown(context.Background())
 	assert.NoError(t, err)
-	assert.True(t, stp.isShutdown)
+	assert.True(t, stp.isShutdown.Load())
 }
 
 func TestSchemaURL(t *testing.T) {
@@ -131,7 +158,7 @@ func TestRegisterAfterShutdownWithoutProcessors(t *testing.T) {
 	stp := NewTracerProvider()
 	err := stp.Shutdown(context.Background())
 	assert.NoError(t, err)
-	assert.True(t, stp.isShutdown)
+	assert.True(t, stp.isShutdown.Load())
 
 	sp := &basicSpanProcessor{}
 	stp.RegisterSpanProcessor(sp) // no-op
@@ -145,7 +172,7 @@ func TestRegisterAfterShutdownWithProcessors(t *testing.T) {
 	stp.RegisterSpanProcessor(sp1)
 	err := stp.Shutdown(context.Background())
 	assert.NoError(t, err)
-	assert.True(t, stp.isShutdown)
+	assert.True(t, stp.isShutdown.Load())
 	assert.Empty(t, stp.spanProcessors.Load().(spanProcessorStates))
 
 	sp2 := &basicSpanProcessor{}