diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt index 364a5e5..1ddbbf6 100644 --- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt +++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt @@ -12,6 +12,7 @@ import android.util.Log import com.alexvas.utils.MediaCodecUtils import com.alexvas.utils.capabilitiesToString import androidx.media3.common.util.Util +import com.limelight.binding.video.MediaCodecHelper import java.nio.ByteBuffer import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean @@ -145,11 +146,15 @@ abstract class VideoDecodeThread ( else Log.i(TAG, "Configuring surface ${safeWidthHeight.first}x${safeWidthHeight.second} w/ '$mimeType'") format.setInteger(MediaFormat.KEY_ROTATION, rotation) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { - // format.setFeatureEnabled(android.media.MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency, true) - // Request low-latency for the decoder. Not all of the decoders support that. - format.setInteger(MediaFormat.KEY_LOW_LATENCY, 1) - } +// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { +// // format.setFeatureEnabled(android.media.MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency, true) +// // Request low-latency for the decoder. Not all of the decoders support that. +// format.setInteger(MediaFormat.KEY_LOW_LATENCY, 1) +// } + + val succeeded = MediaCodecHelper.setDecoderLowLatencyOptions(format, decoder.codecInfo, 1) + Log.i(TAG, "Low-latency: $succeeded") + return format } @@ -391,7 +396,7 @@ abstract class VideoDecodeThread ( if (DEBUG) Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM") break } - } catch (ignored: InterruptedException) { + } catch (_: InterruptedException) { } catch (e: IllegalStateException) { // Restarting decoder in software mode Log.e(TAG, "${e.message}", e) diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspImageView.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspImageView.kt index 1da3ee7..f0b883f 100644 --- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspImageView.kt +++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspImageView.kt @@ -9,6 +9,7 @@ import android.widget.ImageView import com.alexvas.rtsp.codec.VideoDecodeThread import com.alexvas.rtsp.codec.VideoDecoderBitmapThread import com.alexvas.rtsp.widget.RtspProcessor.Statistics +import com.limelight.binding.video.MediaCodecHelper /** * Low latency RTSP stream playback on image view (bitmap). @@ -72,6 +73,7 @@ class RtspImageView : ImageView { private fun initView(context: Context, attrs: AttributeSet?, defStyleAttr: Int) { if (DEBUG) Log.v(TAG, "initView()") + MediaCodecHelper.initialize(context, /*glRenderer*/ "") } fun init(uri: Uri, username: String?, password: String?, userAgent: String?) { diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt index b62056a..0a84145 100644 --- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt +++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt @@ -25,7 +25,6 @@ import org.jcodec.codecs.h264.io.model.VUIParameters import java.net.Socket import java.nio.ByteBuffer import java.util.concurrent.atomic.AtomicBoolean -import kotlin.compareTo import kotlin.math.min class RtspProcessor( diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt index bce6bfd..959315c 100644 --- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt +++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt @@ -12,6 +12,7 @@ import androidx.media3.container.NalUnitUtil import com.alexvas.rtsp.codec.VideoDecodeThread.DecoderType import com.alexvas.rtsp.codec.VideoDecoderSurfaceThread import com.alexvas.rtsp.widget.RtspProcessor.Statistics +import com.limelight.binding.video.MediaCodecHelper /** * Low latency RTSP stream playback on surface view. @@ -88,6 +89,7 @@ open class RtspSurfaceView: SurfaceView { private fun initView(context: Context, attrs: AttributeSet?, defStyleAttr: Int) { if (DEBUG) Log.v(TAG, "initView()") + MediaCodecHelper.initialize(context, /*glRenderer*/ "") holder.addCallback(surfaceCallback) } diff --git a/library-client-rtsp/src/main/java/com/limelight/binding/video/MediaCodecHelper.java b/library-client-rtsp/src/main/java/com/limelight/binding/video/MediaCodecHelper.java new file mode 100644 index 0000000..91ece59 --- /dev/null +++ b/library-client-rtsp/src/main/java/com/limelight/binding/video/MediaCodecHelper.java @@ -0,0 +1,1018 @@ +package com.limelight.binding.video; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import android.annotation.SuppressLint; +import android.app.ActivityManager; +import android.content.Context; +import android.content.pm.ConfigurationInfo; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaCodecInfo.CodecProfileLevel; +import android.media.MediaFormat; +import android.os.Build; +import android.util.Log; + +// Based on https://github.com/moonlight-stream/moonlight-android/blob/master/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java +public class MediaCodecHelper { + + private static final String TAG = MediaCodecHelper.class.getSimpleName(); + + private static final List preferredDecoders; + + private static final List blacklistedDecoderPrefixes; + private static final List spsFixupBitstreamFixupDecoderPrefixes; + private static final List blacklistedAdaptivePlaybackPrefixes; + private static final List baselineProfileHackPrefixes; + private static final List directSubmitPrefixes; + private static final List constrainedHighProfilePrefixes; + private static final List whitelistedHevcDecoders; + private static final List refFrameInvalidationAvcPrefixes; + private static final List refFrameInvalidationHevcPrefixes; + private static final List useFourSlicesPrefixes; + private static final List qualcommDecoderPrefixes; + private static final List kirinDecoderPrefixes; + private static final List exynosDecoderPrefixes; + private static final List amlogicDecoderPrefixes; + private static final List knownVendorLowLatencyOptions; + + public static final boolean SHOULD_BYPASS_SOFTWARE_BLOCK = + Build.HARDWARE.equals("ranchu") || Build.HARDWARE.equals("cheets") || Build.BRAND.equals("Android-x86"); + + private static boolean isLowEndSnapdragon = false; + private static boolean isAdreno620 = false; + private static boolean initialized = false; + + static { + directSubmitPrefixes = new LinkedList<>(); + + // These decoders have low enough input buffer latency that they + // can be directly invoked from the receive thread + directSubmitPrefixes.add("omx.qcom"); + directSubmitPrefixes.add("omx.sec"); + directSubmitPrefixes.add("omx.exynos"); + directSubmitPrefixes.add("omx.intel"); + directSubmitPrefixes.add("omx.brcm"); + directSubmitPrefixes.add("omx.TI"); + directSubmitPrefixes.add("omx.arc"); + directSubmitPrefixes.add("omx.nvidia"); + + // All Codec2 decoders + directSubmitPrefixes.add("c2."); + } + + static { + refFrameInvalidationAvcPrefixes = new LinkedList<>(); + + refFrameInvalidationHevcPrefixes = new LinkedList<>(); + refFrameInvalidationHevcPrefixes.add("omx.exynos"); + refFrameInvalidationHevcPrefixes.add("c2.exynos"); + + // Qualcomm and NVIDIA may be added at runtime + } + + static { + preferredDecoders = new LinkedList<>(); + } + + static { + blacklistedDecoderPrefixes = new LinkedList<>(); + + // Blacklist software decoders that don't support H264 high profile except on systems + // that are expected to only have software decoders (like emulators). + if (!SHOULD_BYPASS_SOFTWARE_BLOCK) { + blacklistedDecoderPrefixes.add("omx.google"); + blacklistedDecoderPrefixes.add("AVCDecoder"); + + // We want to avoid ffmpeg decoders since they're usually software decoders, + // but we'll defer to the Android 10 isSoftwareOnly() API on newer devices + // to determine if we should use these or not. + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { + blacklistedDecoderPrefixes.add("OMX.ffmpeg"); + } + } + + // Force these decoders disabled because: + // 1) They are software decoders, so the performance is terrible + // 2) They crash with our HEVC stream anyway (at least prior to CSD batching) + blacklistedDecoderPrefixes.add("OMX.qcom.video.decoder.hevcswvdec"); + blacklistedDecoderPrefixes.add("OMX.SEC.hevc.sw.dec"); + } + + static { + // If a decoder qualifies for reference frame invalidation, + // these entries will be ignored for those decoders. + spsFixupBitstreamFixupDecoderPrefixes = new LinkedList<>(); + spsFixupBitstreamFixupDecoderPrefixes.add("omx.nvidia"); + spsFixupBitstreamFixupDecoderPrefixes.add("omx.qcom"); + spsFixupBitstreamFixupDecoderPrefixes.add("omx.brcm"); + + baselineProfileHackPrefixes = new LinkedList<>(); + baselineProfileHackPrefixes.add("omx.intel"); + + blacklistedAdaptivePlaybackPrefixes = new LinkedList<>(); + // The Intel decoder on Lollipop on Nexus Player would increase latency badly + // if adaptive playback was enabled so let's avoid it to be safe. + blacklistedAdaptivePlaybackPrefixes.add("omx.intel"); + // The MediaTek decoder crashes at 1080p when adaptive playback is enabled + // on some Android TV devices with HEVC only. + blacklistedAdaptivePlaybackPrefixes.add("omx.mtk"); + + constrainedHighProfilePrefixes = new LinkedList<>(); + constrainedHighProfilePrefixes.add("omx.intel"); + } + + static { + whitelistedHevcDecoders = new LinkedList<>(); + + // Allow software HEVC decoding in the official AOSP emulator + if (Build.HARDWARE.equals("ranchu")) { + whitelistedHevcDecoders.add("omx.google"); + } + + // Exynos seems to be the only HEVC decoder that works reliably + whitelistedHevcDecoders.add("omx.exynos"); + + // On Darcy (Shield 2017), HEVC runs fine with no fixups required. For some reason, + // other X1 implementations require bitstream fixups. However, since numReferenceFrames + // has been supported in GFE since late 2017, we'll go ahead and enable HEVC for all + // device models. + // + // NVIDIA does partial HEVC acceleration on the Shield Tablet. I don't know + // whether the performance is good enough to use for streaming, but they're + // using the same omx.nvidia.h265.decode name as the Shield TV which has a + // fully accelerated HEVC pipeline. AFAIK, the only K1 devices with this + // partially accelerated HEVC decoder are the Shield Tablet and Xiaomi MiPad, + // so I'll check for those here. + // + // In case there are some that I missed, I will also exclude pre-Oreo OSes since + // only Shield ATV got an Oreo update and any newer Tegra devices will not ship + // with an old OS like Nougat. + if (!Build.DEVICE.equalsIgnoreCase("shieldtablet") && + !Build.DEVICE.equalsIgnoreCase("mocha") && + Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + whitelistedHevcDecoders.add("omx.nvidia"); + } + + // Plot twist: On newer Sony devices (BRAVIA_ATV2, BRAVIA_ATV3_4K, BRAVIA_UR1_4K) the H.264 decoder crashes + // on several configurations (> 60 FPS and 1440p) that work with HEVC, so we'll whitelist those devices for HEVC. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && Build.DEVICE.startsWith("BRAVIA_")) { + whitelistedHevcDecoders.add("omx.mtk"); + } + + // Amlogic requires 1 reference frame for HEVC to avoid hanging. Since it's been years + // since GFE added support for maxNumReferenceFrames, we'll just enable all Amlogic SoCs + // running Android 9 or later. + // + // NB: We don't do this on Sabrina (GCWGTV) because H.264 is lower latency when we use + // vendor.low-latency.enable. We will still use HEVC if decoderCanMeetPerformancePointWithHevcAndNotAvc() + // determines it's the only way to meet the performance requirements. + // + // With the Android 12 update, Sabrina now uses HEVC (with RFI) based upon FEATURE_LowLatency + // support, which provides equivalent latency to H.264 now. + // + // FIXME: Should we do this for all Amlogic S905X SoCs? + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && !Build.DEVICE.equalsIgnoreCase("sabrina")) { + whitelistedHevcDecoders.add("omx.amlogic"); + } + + // Realtek SoCs are used inside many Android TV devices and can only do 4K60 with HEVC. + // We'll enable those HEVC decoders by default and see if anything breaks. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + whitelistedHevcDecoders.add("omx.realtek"); + } + + // These theoretically have good HEVC decoding capabilities (potentially better than + // their AVC decoders), but haven't been tested enough + //whitelistedHevcDecoders.add("omx.rk"); + + // Let's see if HEVC decoders are finally stable with C2 + whitelistedHevcDecoders.add("c2."); + + // Based on GPU attributes queried at runtime, the omx.qcom/c2.qti prefix will be added + // during initialization to avoid SoCs with broken HEVC decoders. + } + + static { + useFourSlicesPrefixes = new LinkedList<>(); + + // Software decoders will use 4 slices per frame to allow for slice multithreading + useFourSlicesPrefixes.add("omx.google"); + useFourSlicesPrefixes.add("AVCDecoder"); + useFourSlicesPrefixes.add("omx.ffmpeg"); + useFourSlicesPrefixes.add("c2.android"); + + // Old Qualcomm decoders are detected at runtime + } + + static { + knownVendorLowLatencyOptions = new LinkedList<>(); + + knownVendorLowLatencyOptions.add("vendor.qti-ext-dec-low-latency.enable"); + knownVendorLowLatencyOptions.add("vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req"); + knownVendorLowLatencyOptions.add("vendor.rtc-ext-dec-low-latency.enable"); + knownVendorLowLatencyOptions.add("vendor.low-latency.enable"); + } + + static { + qualcommDecoderPrefixes = new LinkedList<>(); + + qualcommDecoderPrefixes.add("omx.qcom"); + qualcommDecoderPrefixes.add("c2.qti"); + } + + static { + kirinDecoderPrefixes = new LinkedList<>(); + + kirinDecoderPrefixes.add("omx.hisi"); + kirinDecoderPrefixes.add("c2.hisi"); // Unconfirmed + } + + static { + exynosDecoderPrefixes = new LinkedList<>(); + + exynosDecoderPrefixes.add("omx.exynos"); + exynosDecoderPrefixes.add("c2.exynos"); + } + + static { + amlogicDecoderPrefixes = new LinkedList<>(); + + amlogicDecoderPrefixes.add("omx.amlogic"); + amlogicDecoderPrefixes.add("c2.amlogic"); // Unconfirmed + } + + private static boolean isPowerVR(String glRenderer) { + return glRenderer.toLowerCase().contains("powervr"); + } + + private static String getAdrenoVersionString(String glRenderer) { + glRenderer = glRenderer.toLowerCase().trim(); + + if (!glRenderer.contains("adreno")) { + return null; + } + + Pattern modelNumberPattern = Pattern.compile("(.*)([0-9]{3})(.*)"); + + Matcher matcher = modelNumberPattern.matcher(glRenderer); + if (!matcher.matches()) { + return null; + } + + String modelNumber = matcher.group(2); + Log.i(TAG, "Found Adreno GPU: "+modelNumber); + return modelNumber; + } + + private static boolean isLowEndSnapdragonRenderer(String glRenderer) { + String modelNumber = getAdrenoVersionString(glRenderer); + if (modelNumber == null) { + // Not an Adreno GPU + return false; + } + + // The current logic is to identify low-end SoCs based on a zero in the x0x place. + return modelNumber.charAt(1) == '0'; + } + + private static int getAdrenoRendererModelNumber(String glRenderer) { + String modelNumber = getAdrenoVersionString(glRenderer); + if (modelNumber == null) { + // Not an Adreno GPU + return -1; + } + + return Integer.parseInt(modelNumber); + } + + // This is a workaround for some broken devices that report + // only GLES 3.0 even though the GPU is an Adreno 4xx series part. + // An example of such a device is the Huawei Honor 5x with the + // Snapdragon 616 SoC (Adreno 405). + private static boolean isGLES31SnapdragonRenderer(String glRenderer) { + // Snapdragon 4xx and higher support GLES 3.1 + return getAdrenoRendererModelNumber(glRenderer) >= 400; + } + + public static void initialize(Context context, String glRenderer) { + if (initialized) { + return; + } + + // Older Sony ATVs (SVP-DTV15) have broken MediaTek codecs (decoder hangs after rendering the first frame). + // I know the Fire TV 2 and 3 works, so I'll whitelist Amazon devices which seem to actually be tested. + // We still have to check Build.MANUFACTURER to catch Amazon Fire tablets. + if (context.getPackageManager().hasSystemFeature("amazon.hardware.fire_tv") || + Build.MANUFACTURER.equalsIgnoreCase("Amazon")) { + // HEVC and RFI have been confirmed working on Fire TV 2, Fire TV Stick 2, Fire TV 4K Max, + // Fire HD 8 2020, and Fire HD 8 2022 models. + // + // This is probably a good enough sample to conclude that all MediaTek Fire OS devices + // are likely to be okay. + whitelistedHevcDecoders.add("omx.mtk"); + refFrameInvalidationHevcPrefixes.add("omx.mtk"); + refFrameInvalidationHevcPrefixes.add("c2.mtk"); + + // This requires setting vdec-lowlatency on the Fire TV 3, otherwise the decoder + // never produces any output frames. See comment above for details on why we only + // do this for Fire TV devices. + whitelistedHevcDecoders.add("omx.amlogic"); + + // Fire TV 3 seems to produce random artifacts on HEVC streams after packet loss. + // Enabling RFI turns these artifacts into full decoder output hangs, so let's not enable + // that for Fire OS 6 Amlogic devices. We will leave HEVC enabled because that's the only + // way these devices can hit 4K. Hopefully this is just a problem with the BSP used in + // the Fire OS 6 Amlogic devices, so we will leave this enabled for Fire OS 7+. + // + // Apart from a few TV models, the main Amlogic-based Fire TV devices are the Fire TV + // Cubes and Fire TV 3. This check will exclude the Fire TV 3 and Fire TV Cube 1, but + // allow the newer Fire TV Cubes to use HEVC RFI. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + refFrameInvalidationHevcPrefixes.add("omx.amlogic"); + refFrameInvalidationHevcPrefixes.add("c2.amlogic"); + } + } + + ActivityManager activityManager = + (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + ConfigurationInfo configInfo = activityManager.getDeviceConfigurationInfo(); + if (configInfo.reqGlEsVersion != ConfigurationInfo.GL_ES_VERSION_UNDEFINED) { + Log.i(TAG, "OpenGL ES version: "+configInfo.reqGlEsVersion); + + isLowEndSnapdragon = isLowEndSnapdragonRenderer(glRenderer); + isAdreno620 = getAdrenoRendererModelNumber(glRenderer) == 620; + + // Tegra K1 and later can do reference frame invalidation properly + if (configInfo.reqGlEsVersion >= 0x30000) { + Log.i(TAG, "Added omx.nvidia/c2.nvidia to reference frame invalidation support list"); + refFrameInvalidationAvcPrefixes.add("omx.nvidia"); + + // Exclude HEVC RFI on Pixel C and Tegra devices prior to Android 11. Misbehaving RFI + // on these devices can cause hundreds of milliseconds of latency, so it's not worth + // using it unless we're absolutely sure that it will not cause increased latency. + if (!Build.DEVICE.equalsIgnoreCase("dragon") && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + refFrameInvalidationHevcPrefixes.add("omx.nvidia"); + } + + refFrameInvalidationAvcPrefixes.add("c2.nvidia"); // Unconfirmed + refFrameInvalidationHevcPrefixes.add("c2.nvidia"); // Unconfirmed + + Log.i(TAG, "Added omx.qcom/c2.qti to reference frame invalidation support list"); + refFrameInvalidationAvcPrefixes.add("omx.qcom"); + refFrameInvalidationHevcPrefixes.add("omx.qcom"); + refFrameInvalidationAvcPrefixes.add("c2.qti"); + refFrameInvalidationHevcPrefixes.add("c2.qti"); + } + + // Qualcomm's early HEVC decoders break hard on our HEVC stream. The best check to + // tell the good from the bad decoders are the generation of Adreno GPU included: + // 3xx - bad + // 4xx - good + // + // The "good" GPUs support GLES 3.1, but we can't just check that directly + // (see comment on isGLES31SnapdragonRenderer). + // + if (isGLES31SnapdragonRenderer(glRenderer)) { + Log.i(TAG, "Added omx.qcom/c2.qti to HEVC decoders based on GLES 3.1+ support"); + whitelistedHevcDecoders.add("omx.qcom"); + whitelistedHevcDecoders.add("c2.qti"); + } + else { + blacklistedDecoderPrefixes.add("OMX.qcom.video.decoder.hevc"); + + // These older decoders need 4 slices per frame for best performance + useFourSlicesPrefixes.add("omx.qcom"); + } + + // Older MediaTek SoCs have issues with HEVC rendering but the newer chips with + // PowerVR GPUs have good HEVC support. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && isPowerVR(glRenderer)) { + Log.i(TAG, "Added omx.mtk to HEVC decoders based on PowerVR GPU"); + whitelistedHevcDecoders.add("omx.mtk"); + + // This SoC (MT8176 in GPD XD+) supports AVC RFI too, but the maxNumReferenceFrames setting + // required to make it work adds a huge amount of latency. However, RFI on HEVC causes + // decoder hangs on the newer GE8100, GE8300, and GE8320 GPUs, so we limit it to the + // Series6XT GPUs where we know it works. + if (glRenderer.contains("GX6")) { + Log.i(TAG, "Added omx.mtk/c2.mtk to RFI list for HEVC"); + refFrameInvalidationHevcPrefixes.add("omx.mtk"); + refFrameInvalidationHevcPrefixes.add("c2.mtk"); + } + } + } + + initialized = true; + } + + private static boolean isDecoderInList(List decoderList, String decoderName) { + if (!initialized) { + throw new IllegalStateException("MediaCodecHelper must be initialized before use"); + } + + for (String badPrefix : decoderList) { + if (decoderName.length() >= badPrefix.length()) { + String prefix = decoderName.substring(0, badPrefix.length()); + if (prefix.equalsIgnoreCase(badPrefix)) { + return true; + } + } + } + + return false; + } + + private static boolean decoderSupportsAndroidRLowLatency(MediaCodecInfo decoderInfo, String mimeType) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + try { + if (decoderInfo.getCapabilitiesForType(mimeType).isFeatureSupported(CodecCapabilities.FEATURE_LowLatency)) { + Log.i(TAG, "Low latency decoding mode supported (FEATURE_LowLatency)"); + return true; + } + } catch (Exception e) { + // Tolerate buggy codecs + e.printStackTrace(); + } + } + + return false; + } + + private static boolean decoderSupportsKnownVendorLowLatencyOption(String decoderName) { + // It's only possible to probe vendor parameters on Android 12 and above. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + MediaCodec testCodec = null; + try { + // Unfortunately we have to create an actual codec instance to get supported options. + testCodec = MediaCodec.createByCodecName(decoderName); + + // See if any of the vendor parameters match ones we know about + for (String supportedOption : testCodec.getSupportedVendorParameters()) { + for (String knownLowLatencyOption : knownVendorLowLatencyOptions) { + if (supportedOption.equalsIgnoreCase(knownLowLatencyOption)) { + Log.i(TAG, decoderName + " supports known low latency option: " + supportedOption); + return true; + } + } + } + } catch (Exception e) { + // Tolerate buggy codecs + e.printStackTrace(); + } finally { + if (testCodec != null) { + testCodec.release(); + } + } + } + return false; + } + + private static boolean decoderSupportsMaxOperatingRate(String decoderName) { + // Operate at maximum rate to lower latency as much as possible on + // some Qualcomm platforms. We could also set KEY_PRIORITY to 0 (realtime) + // but that will actually result in the decoder crashing if it can't satisfy + // our (ludicrous) operating rate requirement. This seems to cause reliable + // crashes on the Xiaomi Mi 10 lite 5G and Redmi K30i 5G on Android 10, so + // we'll disable it on Snapdragon 765G and all non-Qualcomm devices to be safe. + // + // NB: Even on Android 10, this optimization still provides significant + // performance gains on Pixel 2. + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && + isDecoderInList(qualcommDecoderPrefixes, decoderName) && + !isAdreno620; + } + + public static boolean setDecoderLowLatencyOptions(MediaFormat videoFormat, MediaCodecInfo decoderInfo, int tryNumber) { + // Options here should be tried in the order of most to least risky. The decoder will use + // the first MediaFormat that doesn't fail in configure(). + + boolean setNewOption = false; + + if (tryNumber < 1) { + // Official Android 11+ low latency option (KEY_LOW_LATENCY). + videoFormat.setInteger("low-latency", 1); + setNewOption = true; + + // If this decoder officially supports FEATURE_LowLatency, we will just use that alone + // for try 0. Otherwise, we'll include it as best effort with other options. + if (decoderSupportsAndroidRLowLatency(decoderInfo, videoFormat.getString(MediaFormat.KEY_MIME))) { + return true; + } + } + + if (tryNumber < 2 && + (!Build.MANUFACTURER.equalsIgnoreCase("xiaomi") || Build.VERSION.SDK_INT > Build.VERSION_CODES.M)) { + // MediaTek decoders don't use vendor-defined keys for low latency mode. Instead, they have a modified + // version of AOSP's ACodec.cpp which supports the "vdec-lowlatency" option. This option is passed down + // to the decoder as OMX.MTK.index.param.video.LowLatencyDecode. + // + // This option is also plumbed for Amazon Amlogic-based devices like the Fire TV 3. Not only does it + // reduce latency on Amlogic, it fixes the HEVC bug that causes the decoder to not output any frames. + // Unfortunately, it does the exact opposite for the Xiaomi MITV4-ANSM0, breaking it in the way that + // Fire TV was broken prior to vdec-lowlatency :( + // + // On Fire TV 3, vdec-lowlatency is translated to OMX.amazon.fireos.index.video.lowLatencyDecode. + // + // https://github.com/yuan1617/Framwork/blob/master/frameworks/av/media/libstagefright/ACodec.cpp + // https://github.com/iykex/vendor_mediatek_proprietary_hardware/blob/master/libomx/video/MtkOmxVdecEx/MtkOmxVdecEx.h + videoFormat.setInteger("vdec-lowlatency", 1); + setNewOption = true; + } + + if (tryNumber < 3) { + if (MediaCodecHelper.decoderSupportsMaxOperatingRate(decoderInfo.getName())) { + videoFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, Short.MAX_VALUE); + setNewOption = true; + } + else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + videoFormat.setInteger(MediaFormat.KEY_PRIORITY, 0); + setNewOption = true; + } + } + + // MediaCodec supports vendor-defined format keys using the "vendor.." syntax. + // These allow access to functionality that is not exposed through documented MediaFormat.KEY_* values. + // https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/common/inc/vidc_vendor_extensions.h;l=67 + // + // MediaCodec vendor extension support was introduced in Android 8.0: + // https://cs.android.com/android/_/android/platform/frameworks/av/+/01c10f8cdcd58d1e7025f426a72e6e75ba5d7fc2 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + // Try vendor-specific low latency options + // + // NOTE: Update knownVendorLowLatencyOptions if you modify this code! + if (isDecoderInList(qualcommDecoderPrefixes, decoderInfo.getName())) { + // Examples of Qualcomm's vendor extensions for Snapdragon 845: + // https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp + // https://cs.android.com/android/_/android/platform/hardware/qcom/sm8150/media/+/0621ceb1c1b19564999db8293574a0e12952ff6c + // + // We will first try both, then try vendor.qti-ext-dec-low-latency.enable alone if that fails + if (tryNumber < 4) { + videoFormat.setInteger("vendor.qti-ext-dec-picture-order.enable", 1); + setNewOption = true; + } + if (tryNumber < 5) { + videoFormat.setInteger("vendor.qti-ext-dec-low-latency.enable", 1); + setNewOption = true; + } + } + else if (isDecoderInList(kirinDecoderPrefixes, decoderInfo.getName())) { + if (tryNumber < 4) { + // Kirin low latency options + // https://developer.huawei.com/consumer/cn/forum/topic/0202325564295980115 + videoFormat.setInteger("vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", 1); + videoFormat.setInteger("vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-rdy", -1); + setNewOption = true; + } + } + else if (isDecoderInList(exynosDecoderPrefixes, decoderInfo.getName())) { + if (tryNumber < 4) { + // Exynos low latency option for H.264 decoder + videoFormat.setInteger("vendor.rtc-ext-dec-low-latency.enable", 1); + setNewOption = true; + } + } + else if (isDecoderInList(amlogicDecoderPrefixes, decoderInfo.getName())) { + if (tryNumber < 4) { + // Amlogic low latency vendor extension + // https://github.com/codewalkerster/android_vendor_amlogic_common_prebuilt_libstagefrighthw/commit/41fefc4e035c476d58491324a5fe7666bfc2989e + videoFormat.setInteger("vendor.low-latency.enable", 1); + setNewOption = true; + } + } + } + + return setNewOption; + } + + public static boolean decoderSupportsFusedIdrFrame(MediaCodecInfo decoderInfo, String mimeType) { + // If adaptive playback is supported, we can submit new CSD together with a keyframe + try { + if (decoderInfo.getCapabilitiesForType(mimeType). + isFeatureSupported(CodecCapabilities.FEATURE_AdaptivePlayback)) { + Log.i(TAG, "Decoder supports fused IDR frames (FEATURE_AdaptivePlayback)"); + return true; + } + } catch (Exception e) { + // Tolerate buggy codecs + e.printStackTrace(); + } + + return false; + } + + public static boolean decoderSupportsAdaptivePlayback(MediaCodecInfo decoderInfo, String mimeType) { + if (isDecoderInList(blacklistedAdaptivePlaybackPrefixes, decoderInfo.getName())) { + Log.i(TAG, "Decoder blacklisted for adaptive playback"); + return false; + } + + try { + if (decoderInfo.getCapabilitiesForType(mimeType). + isFeatureSupported(CodecCapabilities.FEATURE_AdaptivePlayback)) + { + // This will make getCapabilities() return that adaptive playback is supported + Log.i(TAG, "Adaptive playback supported (FEATURE_AdaptivePlayback)"); + return true; + } + } catch (Exception e) { + // Tolerate buggy codecs + e.printStackTrace(); + } + + return false; + } + + public static boolean decoderNeedsConstrainedHighProfile(String decoderName) { + return isDecoderInList(constrainedHighProfilePrefixes, decoderName); + } + + public static boolean decoderCanDirectSubmit(String decoderName) { + return isDecoderInList(directSubmitPrefixes, decoderName) && !isExynos4Device(); + } + + public static boolean decoderNeedsSpsBitstreamRestrictions(String decoderName) { + return isDecoderInList(spsFixupBitstreamFixupDecoderPrefixes, decoderName); + } + + public static boolean decoderNeedsBaselineSpsHack(String decoderName) { + return isDecoderInList(baselineProfileHackPrefixes, decoderName); + } + + public static byte getDecoderOptimalSlicesPerFrame(String decoderName) { + if (isDecoderInList(useFourSlicesPrefixes, decoderName)) { + // 4 slices per frame reduces decoding latency on older Qualcomm devices + return 4; + } + else { + // 1 slice per frame produces the optimal encoding efficiency + return 1; + } + } + + public static boolean decoderSupportsRefFrameInvalidationAvc(String decoderName, int videoHeight) { + // Reference frame invalidation is broken on low-end Snapdragon SoCs at 1080p. + if (videoHeight > 720 && isLowEndSnapdragon) { + return false; + } + + // This device seems to crash constantly at 720p, so try disabling + // RFI to see if we can get that under control. + if (Build.DEVICE.equals("b3") || Build.DEVICE.equals("b5")) { + return false; + } + + return isDecoderInList(refFrameInvalidationAvcPrefixes, decoderName); + } + + public static boolean decoderSupportsRefFrameInvalidationHevc(MediaCodecInfo decoderInfo) { + // HEVC decoders seem to universally support RFI, but it can have huge latency penalties + // for some decoders due to the number of references frames being > 1. Old Amlogic + // decoders are known to have this problem. + // + // If the decoder supports FEATURE_LowLatency or any vendor low latency option, + // we will use that as an indication that it can handle HEVC RFI without excessively + // buffering frames. + if (decoderSupportsAndroidRLowLatency(decoderInfo, "video/hevc") || + decoderSupportsKnownVendorLowLatencyOption(decoderInfo.getName())) { + Log.i(TAG, "Enabling HEVC RFI based on low latency option support"); + return true; + } + + return isDecoderInList(refFrameInvalidationHevcPrefixes, decoderInfo.getName()); + } + + public static boolean decoderSupportsRefFrameInvalidationAv1(MediaCodecInfo decoderInfo) { + // We'll use the same heuristics as HEVC for now + if (decoderSupportsAndroidRLowLatency(decoderInfo, "video/av01") || + decoderSupportsKnownVendorLowLatencyOption(decoderInfo.getName())) { + Log.i(TAG, "Enabling AV1 RFI based on low latency option support"); + return true; + } + + return false; + } + + public static boolean decoderIsWhitelistedForHevc(MediaCodecInfo decoderInfo) { + // + // Software decoders are terrible and we never want to use them. + // We want to catch decoders like: + // OMX.qcom.video.decoder.hevcswvdec + // OMX.SEC.hevc.sw.dec + // + if (decoderInfo.getName().contains("sw")) { + Log.i(TAG, "Disallowing HEVC on software decoder: " + decoderInfo.getName()); + return false; + } + else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && (!decoderInfo.isHardwareAccelerated() || decoderInfo.isSoftwareOnly())) { + Log.i(TAG, "Disallowing HEVC on software decoder: " + decoderInfo.getName()); + return false; + } + + // If this device is media performance class 12 or higher, we will assume any hardware + // HEVC decoder present is fast and modern enough for streaming. + // + // [5.3/H-1-1] MUST NOT drop more than 2 frames in 10 seconds (i.e less than 0.333 percent frame drop) for a 1080p 60 fps video session under load. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + Log.i(TAG, "Media performance class: " + Build.VERSION.MEDIA_PERFORMANCE_CLASS); + if (Build.VERSION.MEDIA_PERFORMANCE_CLASS >= Build.VERSION_CODES.S) { + Log.i(TAG, "Allowing HEVC based on media performance class"); + return true; + } + } + + // If the decoder supports FEATURE_LowLatency, we will assume it is fast and modern enough + // to be preferable for streaming over H.264 decoders. + if (decoderSupportsAndroidRLowLatency(decoderInfo, "video/hevc")) { + Log.i(TAG, "Allowing HEVC based on FEATURE_LowLatency support"); + return true; + } + + // Otherwise, we use our list of known working HEVC decoders + return isDecoderInList(whitelistedHevcDecoders, decoderInfo.getName()); + } + + public static boolean isDecoderWhitelistedForAv1(MediaCodecInfo decoderInfo) { + // Google didn't have official support for AV1 (or more importantly, a CTS test) until + // Android 10, so don't use any decoder before then. + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { + return false; + } + + // + // Software decoders are terrible and we never want to use them. + // We want to catch decoders like: + // OMX.qcom.video.decoder.hevcswvdec + // OMX.SEC.hevc.sw.dec + // + if (decoderInfo.getName().contains("sw")) { + Log.i(TAG, "Disallowing AV1 on software decoder: " + decoderInfo.getName()); + return false; + } + else if (!decoderInfo.isHardwareAccelerated() || decoderInfo.isSoftwareOnly()) { + Log.i(TAG, "Disallowing AV1 on software decoder: " + decoderInfo.getName()); + return false; + } + + // TODO: Test some AV1 decoders + return false; + } + + @SuppressWarnings("deprecation") + @SuppressLint("NewApi") + private static LinkedList getMediaCodecList() { + LinkedList infoList = new LinkedList<>(); + + MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); + Collections.addAll(infoList, mcl.getCodecInfos()); + + return infoList; + } + + @SuppressWarnings("RedundantThrows") + public static String dumpDecoders() throws Exception { + String str = ""; + for (MediaCodecInfo codecInfo : getMediaCodecList()) { + // Skip encoders + if (codecInfo.isEncoder()) { + continue; + } + + str += "Decoder: "+codecInfo.getName()+"\n"; + for (String type : codecInfo.getSupportedTypes()) { + str += "\t"+type+"\n"; + CodecCapabilities caps = codecInfo.getCapabilitiesForType(type); + + for (CodecProfileLevel profile : caps.profileLevels) { + str += "\t\t"+profile.profile+" "+profile.level+"\n"; + } + } + } + return str; + } + + private static MediaCodecInfo findPreferredDecoder() { + // This is a different algorithm than the other findXXXDecoder functions, + // because we want to evaluate the decoders in our list's order + // rather than MediaCodecList's order + + if (!initialized) { + throw new IllegalStateException("MediaCodecHelper must be initialized before use"); + } + + for (String preferredDecoder : preferredDecoders) { + for (MediaCodecInfo codecInfo : getMediaCodecList()) { + // Skip encoders + if (codecInfo.isEncoder()) { + continue; + } + + // Check for preferred decoders + if (preferredDecoder.equalsIgnoreCase(codecInfo.getName())) { + Log.i(TAG, "Preferred decoder choice is "+codecInfo.getName()); + return codecInfo; + } + } + } + + return null; + } + + private static boolean isCodecBlacklisted(MediaCodecInfo codecInfo) { + // Use the new isSoftwareOnly() function on Android Q + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + if (!SHOULD_BYPASS_SOFTWARE_BLOCK && codecInfo.isSoftwareOnly()) { + Log.i(TAG, "Skipping software-only decoder: "+codecInfo.getName()); + return true; + } + } + + // Check for explicitly blacklisted decoders + if (isDecoderInList(blacklistedDecoderPrefixes, codecInfo.getName())) { + Log.i(TAG, "Skipping blacklisted decoder: "+codecInfo.getName()); + return true; + } + + return false; + } + + public static MediaCodecInfo findFirstDecoder(String mimeType) { + for (MediaCodecInfo codecInfo : getMediaCodecList()) { + // Skip encoders + if (codecInfo.isEncoder()) { + continue; + } + + // Skip compatibility aliases on Q+ + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + if (codecInfo.isAlias()) { + continue; + } + } + + // Find a decoder that supports the specified video format + for (String mime : codecInfo.getSupportedTypes()) { + if (mime.equalsIgnoreCase(mimeType)) { + // Skip blacklisted codecs + if (isCodecBlacklisted(codecInfo)) { + continue; + } + + Log.i(TAG, "First decoder choice is "+codecInfo.getName()); + return codecInfo; + } + } + } + + return null; + } + + public static MediaCodecInfo findProbableSafeDecoder(String mimeType, int requiredProfile) { + // First look for a preferred decoder by name + MediaCodecInfo info = findPreferredDecoder(); + if (info != null) { + return info; + } + + // Now look for decoders we know are safe + try { + // If this function completes, it will determine if the decoder is safe + return findKnownSafeDecoder(mimeType, requiredProfile); + } catch (Exception e) { + // Some buggy devices seem to throw exceptions + // from getCapabilitiesForType() so we'll just assume + // they're okay and go with the first one we find + return findFirstDecoder(mimeType); + } + } + + // We declare this method as explicitly throwing Exception + // since some bad decoders can throw IllegalArgumentExceptions unexpectedly + // and we want to be sure all callers are handling this possibility + @SuppressWarnings("RedundantThrows") + private static MediaCodecInfo findKnownSafeDecoder(String mimeType, int requiredProfile) throws Exception { + // Some devices (Exynos devces, at least) have two sets of decoders. + // The first set of decoders are C2 which do not support FEATURE_LowLatency, + // but the second set of OMX decoders do support FEATURE_LowLatency. We want + // to pick the OMX decoders despite the fact that C2 is listed first. + // On some Qualcomm devices (like Pixel 4), there are separate low latency decoders + // (like c2.qti.hevc.decoder.low_latency) that advertise FEATURE_LowLatency while + // the standard ones (like c2.qti.hevc.decoder) do not. Like Exynos, the decoders + // with FEATURE_LowLatency support are listed after the standard ones. + for (int i = 0; i < 2; i++) { + for (MediaCodecInfo codecInfo : getMediaCodecList()) { + // Skip encoders + if (codecInfo.isEncoder()) { + continue; + } + + // Skip compatibility aliases on Q+ + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + if (codecInfo.isAlias()) { + continue; + } + } + + // Find a decoder that supports the requested video format + for (String mime : codecInfo.getSupportedTypes()) { + if (mime.equalsIgnoreCase(mimeType)) { + Log.i(TAG, "Examining decoder capabilities of " + codecInfo.getName() + " (round " + (i + 1) + ")"); + + // Skip blacklisted codecs + if (isCodecBlacklisted(codecInfo)) { + continue; + } + + CodecCapabilities caps = codecInfo.getCapabilitiesForType(mime); + + if (i == 0 && !decoderSupportsAndroidRLowLatency(codecInfo, mime)) { + Log.i(TAG, "Skipping decoder that lacks FEATURE_LowLatency for round 1"); + continue; + } + + if (requiredProfile != -1) { + for (CodecProfileLevel profile : caps.profileLevels) { + if (profile.profile == requiredProfile) { + Log.i(TAG, "Decoder " + codecInfo.getName() + " supports required profile"); + return codecInfo; + } + } + + Log.i(TAG, "Decoder " + codecInfo.getName() + " does NOT support required profile"); + } else { + return codecInfo; + } + } + } + } + } + + return null; + } + + public static String readCpuinfo() throws Exception { + StringBuilder cpuInfo = new StringBuilder(); + try (final BufferedReader br = new BufferedReader(new FileReader(new File("/proc/cpuinfo")))) { + for (;;) { + int ch = br.read(); + if (ch == -1) + break; + cpuInfo.append((char)ch); + } + + return cpuInfo.toString(); + } + } + + private static boolean stringContainsIgnoreCase(String string, String substring) { + return string.toLowerCase(Locale.ENGLISH).contains(substring.toLowerCase(Locale.ENGLISH)); + } + + public static boolean isExynos4Device() { + try { + // Try reading CPU info too look for + String cpuInfo = readCpuinfo(); + + // SMDK4xxx is Exynos 4 + if (stringContainsIgnoreCase(cpuInfo, "SMDK4")) { + Log.i(TAG, "Found SMDK4 in /proc/cpuinfo"); + return true; + } + + // If we see "Exynos 4" also we'll count it + if (stringContainsIgnoreCase(cpuInfo, "Exynos 4")) { + Log.i(TAG, "Found Exynos 4 in /proc/cpuinfo"); + return true; + } + } catch (Exception e) { + e.printStackTrace(); + } + + try { + File systemDir = new File("/sys/devices/system"); + File[] files = systemDir.listFiles(); + if (files != null) { + for (File f : files) { + if (stringContainsIgnoreCase(f.getName(), "exynos4")) { + Log.i(TAG, "Found exynos4 in /sys/devices/system"); + return true; + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + return false; + } +}