Skip to content

Commit

Permalink
TracerProvider allows calling Tracer() while it's shutting down (#3924)
Browse files Browse the repository at this point in the history
  • Loading branch information
ash2k authored Mar 28, 2023
1 parent ae90c44 commit c4940f3
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 22 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm

## [Unreleased]

### Fixed

- `TracerProvider` allows calling `Tracer()` while it's shutting down.
It used to deadlock. (#3924)

## [1.15.0-rc.2/0.38.0-rc.2] 2023-03-23

This is a release candidate for the v1.15.0/v0.38.0 release.
Expand Down
65 changes: 50 additions & 15 deletions sdk/trace/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ type TracerProvider struct {
mu sync.Mutex
namedTracer map[instrumentation.Scope]*tracer
spanProcessors atomic.Pointer[spanProcessorStates]
isShutdown bool

isShutdown atomic.Bool

// These fields are not protected by the lock mu. They are assumed to be
// immutable after creation of the TracerProvider.
Expand Down Expand Up @@ -136,10 +137,11 @@ func NewTracerProvider(opts ...TracerProviderOption) *TracerProvider {
//
// This method is safe to be called concurrently.
func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer {
// This check happens before the mutex is acquired to avoid deadlocking if Tracer() is called from within Shutdown().
if p.isShutdown.Load() {
return trace.NewNoopTracerProvider().Tracer(name, opts...)
}
c := trace.NewTracerConfig(opts...)

p.mu.Lock()
defer p.mu.Unlock()
if name == "" {
name = defaultTracerName
}
Expand All @@ -148,23 +150,46 @@ func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.T
Version: c.InstrumentationVersion(),
SchemaURL: c.SchemaURL(),
}
t, ok := p.namedTracer[is]
if !ok {
t = &tracer{
provider: p,
instrumentationScope: is,

t, ok := func() (trace.Tracer, bool) {
p.mu.Lock()
defer p.mu.Unlock()
// Must check the flag after acquiring the mutex to avoid returning a valid tracer if Shutdown() ran
// after the first check above but before we acquired the mutex.
if p.isShutdown.Load() {
return trace.NewNoopTracerProvider().Tracer(name, opts...), true
}
t, ok := p.namedTracer[is]
if !ok {
t = &tracer{
provider: p,
instrumentationScope: is,
}
p.namedTracer[is] = t
}
p.namedTracer[is] = t
global.Info("Tracer created", "name", name, "version", c.InstrumentationVersion(), "schemaURL", c.SchemaURL())
return t, ok
}()
if !ok {
// This code is outside the mutex to not hold the lock while calling third party logging code:
// - That code may do slow things like I/O, which would prolong the duration the lock is held,
// slowing down all tracing consumers.
// - Logging code may be instrumented with tracing and deadlock because it could try
// acquiring the same non-reentrant mutex.
global.Info("Tracer created", "name", name, "version", is.Version, "schemaURL", is.SchemaURL)
}
return t
}

// RegisterSpanProcessor adds the given SpanProcessor to the list of SpanProcessors.
func (p *TracerProvider) RegisterSpanProcessor(sp SpanProcessor) {
// This check prevents calls during a shutdown.
if p.isShutdown.Load() {
return
}
p.mu.Lock()
defer p.mu.Unlock()
if p.isShutdown {
// This check prevents calls after a shutdown.
if p.isShutdown.Load() {
return
}
newSPS := spanProcessorStates{}
Expand All @@ -175,9 +200,14 @@ func (p *TracerProvider) RegisterSpanProcessor(sp SpanProcessor) {

// UnregisterSpanProcessor removes the given SpanProcessor from the list of SpanProcessors.
func (p *TracerProvider) UnregisterSpanProcessor(sp SpanProcessor) {
// This check prevents calls during a shutdown.
if p.isShutdown.Load() {
return
}
p.mu.Lock()
defer p.mu.Unlock()
if p.isShutdown {
// This check prevents calls after a shutdown.
if p.isShutdown.Load() {
return
}
old := *(p.spanProcessors.Load())
Expand Down Expand Up @@ -236,13 +266,18 @@ func (p *TracerProvider) ForceFlush(ctx context.Context) error {

// Shutdown shuts down TracerProvider. All registered span processors are shut down
// in the order they were registered and any held computational resources are released.
// After Shutdown is called, all methods are no-ops.
func (p *TracerProvider) Shutdown(ctx context.Context) error {
// This check prevents deadlocks in case of recursive shutdown.
if p.isShutdown.Load() {
return nil
}
p.mu.Lock()
defer p.mu.Unlock()
if p.isShutdown {
// This check prevents calls after a shutdown has already been done concurrently.
if !p.isShutdown.CompareAndSwap(false, true) { // did toggle?
return nil
}
p.isShutdown = true
spss := *(p.spanProcessors.Load())

var retErr error
Expand Down
41 changes: 34 additions & 7 deletions sdk/trace/provider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,38 @@ func (t *basicSpanProcessor) ForceFlush(context.Context) error {
return nil
}

type shutdownSpanProcessor struct {
shutdown func(context.Context) error
}

func (t *shutdownSpanProcessor) Shutdown(ctx context.Context) error {
return t.shutdown(ctx)
}

func (t *shutdownSpanProcessor) OnStart(context.Context, ReadWriteSpan) {}
func (t *shutdownSpanProcessor) OnEnd(ReadOnlySpan) {}
func (t *shutdownSpanProcessor) ForceFlush(context.Context) error {
return nil
}

func TestShutdownCallsTracerMethod(t *testing.T) {
stp := NewTracerProvider()
sp := &shutdownSpanProcessor{
shutdown: func(ctx context.Context) error {
_ = stp.Tracer("abc") // must not deadlock
return nil
},
}
stp.RegisterSpanProcessor(sp)
assert.NoError(t, stp.Shutdown(context.Background()))
assert.True(t, stp.isShutdown.Load())
}

func TestForceFlushAndShutdownTraceProviderWithoutProcessor(t *testing.T) {
stp := NewTracerProvider()
assert.NoError(t, stp.ForceFlush(context.Background()))
assert.NoError(t, stp.Shutdown(context.Background()))
assert.True(t, stp.isShutdown)
assert.True(t, stp.isShutdown.Load())
}

func TestShutdownTraceProvider(t *testing.T) {
Expand All @@ -61,7 +88,7 @@ func TestShutdownTraceProvider(t *testing.T) {
assert.NoError(t, stp.ForceFlush(context.Background()))
assert.True(t, sp.flushed, "error ForceFlush basicSpanProcessor")
assert.NoError(t, stp.Shutdown(context.Background()))
assert.True(t, stp.isShutdown)
assert.True(t, stp.isShutdown.Load())
assert.True(t, sp.closed, "error Shutdown basicSpanProcessor")
}

Expand All @@ -76,7 +103,7 @@ func TestFailedProcessorShutdown(t *testing.T) {
err := stp.Shutdown(context.Background())
assert.Error(t, err)
assert.Equal(t, err, spErr)
assert.True(t, stp.isShutdown)
assert.True(t, stp.isShutdown.Load())
}

func TestFailedProcessorsShutdown(t *testing.T) {
Expand All @@ -97,7 +124,7 @@ func TestFailedProcessorsShutdown(t *testing.T) {
assert.EqualError(t, err, "basic span processor shutdown failure1; basic span processor shutdown failure2")
assert.True(t, sp1.closed)
assert.True(t, sp2.closed)
assert.True(t, stp.isShutdown)
assert.True(t, stp.isShutdown.Load())
}

func TestFailedProcessorShutdownInUnregister(t *testing.T) {
Expand All @@ -114,7 +141,7 @@ func TestFailedProcessorShutdownInUnregister(t *testing.T) {

err := stp.Shutdown(context.Background())
assert.NoError(t, err)
assert.True(t, stp.isShutdown)
assert.True(t, stp.isShutdown.Load())
}

func TestSchemaURL(t *testing.T) {
Expand All @@ -131,7 +158,7 @@ func TestRegisterAfterShutdownWithoutProcessors(t *testing.T) {
stp := NewTracerProvider()
err := stp.Shutdown(context.Background())
assert.NoError(t, err)
assert.True(t, stp.isShutdown)
assert.True(t, stp.isShutdown.Load())

sp := &basicSpanProcessor{}
stp.RegisterSpanProcessor(sp) // no-op
Expand All @@ -145,7 +172,7 @@ func TestRegisterAfterShutdownWithProcessors(t *testing.T) {
stp.RegisterSpanProcessor(sp1)
err := stp.Shutdown(context.Background())
assert.NoError(t, err)
assert.True(t, stp.isShutdown)
assert.True(t, stp.isShutdown.Load())
assert.Empty(t, stp.spanProcessors.Load())

sp2 := &basicSpanProcessor{}
Expand Down

0 comments on commit c4940f3

Please sign in to comment.