From c8aac24ffd8bfe708d68a251a9f28b3b48bed50c Mon Sep 17 00:00:00 2001 From: tonihei Date: Tue, 3 Oct 2023 04:25:47 -0700 Subject: [PATCH] Explicitly mark DecoderOutputBuffer as shouldBeSkipped if needed In some cases, SimpleDecoder output needs to be skipped for rendering because the decoder produces no data. This is one of the remaining usages of BUFFER_FLAG_DECODE_ONLY at the moment and can be more directly solved without using the flag. SimpleDecoder still needs to check the flag though for backwards compatbility with custom decoders while the flag is not completely removed. PiperOrigin-RevId: 570345233 --- RELEASENOTES.md | 3 +++ .../java/androidx/media3/decoder/Buffer.java | 2 ++ .../media3/decoder/DecoderOutputBuffer.java | 16 ++++++++++++++++ .../androidx/media3/decoder/SimpleDecoder.java | 3 ++- .../androidx/media3/decoder/av1/Gav1Decoder.java | 6 +++--- .../decoder/ffmpeg/FfmpegAudioDecoder.java | 6 +++--- .../media3/decoder/opus/OpusDecoder.java | 2 +- .../androidx/media3/decoder/vp9/VpxDecoder.java | 8 +++++--- 8 files changed, 35 insertions(+), 11 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 41836c2648e..56a597d9169 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -29,6 +29,9 @@ * Smooth Streaming Extension: * RTSP Extension: * Decoder Extensions (FFmpeg, VP9, AV1, etc.): + * Add `DecoderOutputBuffer.shouldBeSkipped` to directly mark output + buffers that don't need to be presented. This is preferred over + `C.BUFFER_FLAG_DECODE_ONLY`. * MIDI extension: * Leanback extension: * Cast Extension: diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java index 5c4b0e0dbd2..6e2dbeb581f 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java @@ -15,6 +15,7 @@ */ package androidx.media3.decoder; +import androidx.annotation.CallSuper; import androidx.media3.common.C; import androidx.media3.common.util.UnstableApi; @@ -25,6 +26,7 @@ public abstract class Buffer { private @C.BufferFlags int flags; /** Clears the buffer. */ + @CallSuper public void clear() { flags = 0; } diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/DecoderOutputBuffer.java b/libraries/decoder/src/main/java/androidx/media3/decoder/DecoderOutputBuffer.java index 1bca3cb8173..b2038801aa0 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/DecoderOutputBuffer.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/DecoderOutputBuffer.java @@ -15,6 +15,7 @@ */ package androidx.media3.decoder; +import androidx.annotation.CallSuper; import androidx.media3.common.util.UnstableApi; /** Output buffer decoded by a {@link Decoder}. */ @@ -40,6 +41,21 @@ public interface Owner { */ public int skippedOutputBufferCount; + /** + * Whether this buffer should be skipped, usually because the decoding process generated no data + * or invalid data. + */ + public boolean shouldBeSkipped; + /** Releases the output buffer for reuse. Must be called when the buffer is no longer needed. */ public abstract void release(); + + @Override + @CallSuper + public void clear() { + super.clear(); + timeUs = 0; + skippedOutputBufferCount = 0; + shouldBeSkipped = false; + } } diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java b/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java index 58d2943da03..d4084a904e5 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java @@ -232,6 +232,7 @@ private boolean decode() throws InterruptedException { if (inputBuffer.isEndOfStream()) { outputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); } else { + outputBuffer.timeUs = inputBuffer.timeUs; if (inputBuffer.isDecodeOnly()) { outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); } @@ -262,7 +263,7 @@ private boolean decode() throws InterruptedException { synchronized (lock) { if (flushed) { outputBuffer.release(); - } else if (outputBuffer.isDecodeOnly()) { + } else if (outputBuffer.isDecodeOnly() || outputBuffer.shouldBeSkipped) { skippedOutputBufferCount++; outputBuffer.release(); } else { diff --git a/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java b/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java index 2a93e0fb6f6..49feb4e68cf 100644 --- a/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java +++ b/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java @@ -117,7 +117,7 @@ protected Gav1DecoderException decode( "gav1GetFrame error: " + gav1GetErrorMessage(gav1DecoderContext)); } if (getFrameResult == GAV1_DECODE_ONLY) { - outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); + outputBuffer.shouldBeSkipped = true; } if (!decodeOnly) { outputBuffer.format = inputBuffer.format; @@ -139,9 +139,9 @@ public void release() { @Override protected void releaseOutputBuffer(VideoDecoderOutputBuffer outputBuffer) { - // Decode only frames do not acquire a reference on the internal decoder buffer and thus do not + // Skipped frames do not acquire a reference on the internal decoder buffer and thus do not // require a call to gav1ReleaseFrame. - if (outputBuffer.mode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !outputBuffer.isDecodeOnly()) { + if (outputBuffer.mode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !outputBuffer.shouldBeSkipped) { gav1ReleaseFrame(gav1DecoderContext, outputBuffer); } super.releaseOutputBuffer(outputBuffer); diff --git a/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/FfmpegAudioDecoder.java b/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/FfmpegAudioDecoder.java index 0e072874001..6fc34e71917 100644 --- a/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/FfmpegAudioDecoder.java +++ b/libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/FfmpegAudioDecoder.java @@ -113,13 +113,13 @@ protected FfmpegDecoderException decode( return new FfmpegDecoderException("Error decoding (see logcat)."); } else if (result == AUDIO_DECODER_ERROR_INVALID_DATA) { // Treat invalid data errors as non-fatal to match the behavior of MediaCodec. No output will - // be produced for this buffer, so mark it as decode-only to ensure that the audio sink's + // be produced for this buffer, so mark it as skipped to ensure that the audio sink's // position is reset when more audio is produced. - outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); + outputBuffer.shouldBeSkipped = true; return null; } else if (result == 0) { // There's no need to output empty buffers. - outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); + outputBuffer.shouldBeSkipped = true; return null; } if (!hasOutputFormat) { diff --git a/libraries/decoder_opus/src/main/java/androidx/media3/decoder/opus/OpusDecoder.java b/libraries/decoder_opus/src/main/java/androidx/media3/decoder/opus/OpusDecoder.java index a381b49a4c1..d5637ff2010 100644 --- a/libraries/decoder_opus/src/main/java/androidx/media3/decoder/opus/OpusDecoder.java +++ b/libraries/decoder_opus/src/main/java/androidx/media3/decoder/opus/OpusDecoder.java @@ -229,7 +229,7 @@ protected OpusDecoderException decode( int skipBytes = skipSamples * bytesPerSample; if (result <= skipBytes) { skipSamples -= result / bytesPerSample; - outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); + outputBuffer.shouldBeSkipped = true; outputData.position(result); } else { skipSamples = 0; diff --git a/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java b/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java index 30bdaecb7f4..db85283ec63 100644 --- a/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java +++ b/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java @@ -102,9 +102,9 @@ protected VideoDecoderOutputBuffer createOutputBuffer() { @Override protected void releaseOutputBuffer(VideoDecoderOutputBuffer outputBuffer) { - // Decode only frames do not acquire a reference on the internal decoder buffer and thus do not + // Skipped frames do not acquire a reference on the internal decoder buffer and thus do not // require a call to vpxReleaseFrame. - if (outputMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !outputBuffer.isDecodeOnly()) { + if (outputMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !outputBuffer.shouldBeSkipped) { vpxReleaseFrame(vpxDecContext, outputBuffer); } super.releaseOutputBuffer(outputBuffer); @@ -169,11 +169,13 @@ protected VpxDecoderException decode( outputBuffer.init(inputBuffer.timeUs, outputMode, lastSupplementalData); int getFrameResult = vpxGetFrame(vpxDecContext, outputBuffer); if (getFrameResult == 1) { - outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); + outputBuffer.shouldBeSkipped = true; } else if (getFrameResult == -1) { return new VpxDecoderException("Buffer initialization failed."); } outputBuffer.format = inputBuffer.format; + } else { + outputBuffer.shouldBeSkipped = true; } return null; }