-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Prevent ANR during SDK initialization
Description When initializing the OpenTelemetry Android SDK with disk buffering enabled, we discovered that synchronous disk space checks were causing ANRs in production. These checks occur during the creation of disk buffering exporters, specifically in `DiskManager.getMaxFolderSize()`, which makes blocking IPC calls through `StorageManager.getAllocatableBytes()` on the main thread. The issue manifests in the following ANR stacktrace: ``` android.os.BinderProxy.transact (BinderProxy.java:662) android.os.storage.IStorageManager$Stub$Proxy.getAllocatableBytes (IStorageManager.java:2837) android.os.storage.StorageManager.getAllocatableBytes (StorageManager.java:2414) android.os.storage.StorageManager.getAllocatableBytes (StorageManager.java:2404) io.opentelemetry.android.internal.services.CacheStorage.getAvailableSpace (CacheStorage.java:66) io.opentelemetry.android.internal.services.CacheStorage.ensureCacheSpaceAvailable (CacheStorage.java:50) io.opentelemetry.android.internal.features.persistence.DiskManager.getMaxFolderSize (DiskManager.kt:58) io.opentelemetry.android.OpenTelemetryRumBuilder.createStorageConfiguration (OpenTelemetryRumBuilder.java:338) io.opentelemetry.android.OpenTelemetryRumBuilder.build (OpenTelemetryRumBuilder.java:286) ``` Our Solution To fix this we moved initialization to run on a background executor and buffer the data in memory until it completes. The process works like this: 1. Initialize the SDK with `BufferDelegatingExporter` instances that can immediately accept telemetry data. 2. Move exporter initialization off the main thread. 3. Once async initialization completes, flush buffered signals to initialized exporters and delegate all future signals. The primary goal of this solution is to be unobtrusive and prevent ANRs caused by initialization of disk exporters, while preventing signals from being dropped. Testing We have added unit tests to cover the buffering, delevation, and RUM building. We've also verified this with both disk enabled and disk disabled.
- Loading branch information
Showing
7 changed files
with
527 additions
and
71 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
core/src/main/java/io/opentelemetry/android/export/BufferDelegatingLogExporter.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright The OpenTelemetry Authors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package io.opentelemetry.android.export | ||
|
|
||
| import io.opentelemetry.sdk.common.CompletableResultCode | ||
| import io.opentelemetry.sdk.logs.data.LogRecordData | ||
| import io.opentelemetry.sdk.logs.export.LogRecordExporter | ||
|
|
||
| /** | ||
| * An in-memory buffer delegating log exporter that buffers log records in memory until a delegate is set. | ||
| * Once a delegate is set, the buffered log records are exported to the delegate. | ||
| * | ||
| * The buffer size is set to 5,000 log entries by default. If the buffer is full, the exporter will drop new log records. | ||
| */ | ||
| internal class BufferDelegatingLogExporter( | ||
| maxBufferedLogs: Int = 5_000, | ||
| ) : BufferedDelegatingExporter<LogRecordData, LogRecordExporter>(bufferedSignals = maxBufferedLogs), | ||
| LogRecordExporter { | ||
| override fun exportToDelegate( | ||
| delegate: LogRecordExporter, | ||
| data: Collection<LogRecordData>, | ||
| ): CompletableResultCode = delegate.export(data) | ||
|
|
||
| override fun shutdownDelegate(delegate: LogRecordExporter): CompletableResultCode = delegate.shutdown() | ||
|
|
||
| override fun export(logs: Collection<LogRecordData>): CompletableResultCode = bufferOrDelegate(logs) | ||
|
|
||
| override fun flush(): CompletableResultCode = | ||
| withDelegateOrNull { delegate -> | ||
| delegate?.flush() ?: CompletableResultCode.ofSuccess() | ||
| } | ||
| } |
35 changes: 35 additions & 0 deletions
35
core/src/main/java/io/opentelemetry/android/export/BufferDelegatingSpanExporter.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright The OpenTelemetry Authors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package io.opentelemetry.android.export | ||
|
|
||
| import io.opentelemetry.sdk.common.CompletableResultCode | ||
| import io.opentelemetry.sdk.trace.data.SpanData | ||
| import io.opentelemetry.sdk.trace.export.SpanExporter | ||
|
|
||
| /** | ||
| * An in-memory buffer delegating span exporter that buffers span data in memory until a delegate is set. | ||
| * Once a delegate is set, the buffered span data is exported to the delegate. | ||
| * | ||
| * The buffer size is set to 5,000 spans by default. If the buffer is full, the exporter will drop new span data. | ||
| */ | ||
| internal class BufferDelegatingSpanExporter( | ||
| maxBufferedSpans: Int = 5_000, | ||
| ) : BufferedDelegatingExporter<SpanData, SpanExporter>(bufferedSignals = maxBufferedSpans), | ||
| SpanExporter { | ||
| override fun exportToDelegate( | ||
| delegate: SpanExporter, | ||
| data: Collection<SpanData>, | ||
| ): CompletableResultCode = delegate.export(data) | ||
|
|
||
| override fun shutdownDelegate(delegate: SpanExporter): CompletableResultCode = delegate.shutdown() | ||
|
|
||
| override fun export(spans: Collection<SpanData>): CompletableResultCode = bufferOrDelegate(spans) | ||
|
|
||
| override fun flush(): CompletableResultCode = | ||
| withDelegateOrNull { delegate -> | ||
| delegate?.flush() ?: CompletableResultCode.ofSuccess() | ||
| } | ||
| } |
100 changes: 100 additions & 0 deletions
100
core/src/main/java/io/opentelemetry/android/export/BufferedDelegatingExporter.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| /* | ||
| * Copyright The OpenTelemetry Authors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package io.opentelemetry.android.export | ||
|
|
||
| import io.opentelemetry.sdk.common.CompletableResultCode | ||
| import java.util.concurrent.atomic.AtomicBoolean | ||
|
|
||
| /** | ||
| * An in-memory buffer delegating signal exporter that buffers signal in memory until a delegate is set. | ||
| * Once a delegate is set, the buffered signals are exported to the delegate. | ||
| * | ||
| * The buffer size is set to 5,000 by default. If the buffer is full, the exporter will drop new signals. | ||
| */ | ||
| internal abstract class BufferedDelegatingExporter<T, D>(private val bufferedSignals: Int = 5_000) { | ||
| @Volatile | ||
| private var delegate: D? = null | ||
| private val buffer = arrayListOf<T>() | ||
| private val lock = Any() | ||
| private var isShutDown = AtomicBoolean(false) | ||
|
|
||
| /** | ||
| * Sets the delegate for this exporter and flushes the buffer to the delegate. | ||
| * | ||
| * If the delegate has already been set, an [IllegalStateException] will be thrown. | ||
| * If this exporter has been shut down, the delegate will be shut down immediately. | ||
| * | ||
| * @param delegate the delegate to set | ||
| * | ||
| * @throws IllegalStateException if a delegate has already been set | ||
| */ | ||
| fun setDelegate(delegate: D) { | ||
| synchronized(lock) { | ||
| check(this.delegate == null) { "Exporter delegate has already been set." } | ||
|
|
||
| flushToDelegate(delegate) | ||
|
|
||
| this.delegate = delegate | ||
|
|
||
| if (isShutDown.get()) { | ||
| shutdownDelegate(delegate) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Buffers the given data if the delegate has not been set, otherwise exports the data to the delegate. | ||
| * | ||
| * @param data the data to buffer or export | ||
| */ | ||
| protected fun bufferOrDelegate(data: Collection<T>): CompletableResultCode = | ||
| withDelegateOrNull { | ||
| if (it != null) { | ||
| exportToDelegate(it, data) | ||
| } else { | ||
| val amountToTake = bufferedSignals - buffer.size | ||
| buffer.addAll(data.take(amountToTake)) | ||
| CompletableResultCode.ofSuccess() | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Executes the given block with the delegate if it has been set, otherwise executes the block with a null delegate. | ||
| * | ||
| * @param block the block to execute | ||
| */ | ||
| protected fun <R> withDelegateOrNull(block: (D?) -> R): R { | ||
| delegate?.let { return block(it) } | ||
| return synchronized(lock) { block(delegate) } | ||
| } | ||
|
|
||
| open fun shutdown(): CompletableResultCode = bufferedShutDown() | ||
|
|
||
| protected abstract fun exportToDelegate( | ||
| delegate: D, | ||
| data: Collection<T>, | ||
| ): CompletableResultCode | ||
|
|
||
| protected abstract fun shutdownDelegate(delegate: D): CompletableResultCode | ||
|
|
||
| private fun flushToDelegate(delegate: D) { | ||
| exportToDelegate(delegate, buffer) | ||
| buffer.clear() | ||
| buffer.trimToSize() | ||
| } | ||
|
|
||
| private fun bufferedShutDown(): CompletableResultCode { | ||
| isShutDown.set(true) | ||
|
|
||
| return withDelegateOrNull { | ||
| if (it != null) { | ||
| shutdownDelegate(it) | ||
| } else { | ||
| CompletableResultCode.ofSuccess() | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.