From ae097dee39703f02d908dc73f34041daec3577c5 Mon Sep 17 00:00:00 2001
From: Alexey Vasilyev <alexey.vasilyev@gmail.com>
Date: Sun, 10 Nov 2024 21:36:08 +0200
Subject: [PATCH] Added modification SPS frame w/ low latency params

---
 app/build.gradle                              |  22 ++-
 .../alexvas/rtsp/demo/live/LiveFragment.kt    |   4 +
 app/src/main/res/layout/fragment_live.xml     |   8 +
 build.gradle                                  |   6 +-
 gradle.properties                             |   1 -
 library-client-rtsp/build.gradle              |   5 +-
 .../java/com/alexvas/rtsp/RtspClient.java     |   4 +-
 .../com/alexvas/rtsp/widget/RtspProcessor.kt  | 159 +++++++++++++++---
 .../alexvas/rtsp/widget/RtspSurfaceView.kt    |   4 +
 .../java/com/alexvas/utils/VideoCodecUtils.kt |   3 +
 10 files changed, 179 insertions(+), 37 deletions(-)

diff --git a/app/build.gradle b/app/build.gradle
index 10df154..08c2b45 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -41,16 +41,20 @@ dependencies {
     implementation fileTree(dir: 'libs', include: ['*.jar'])
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
     implementation 'androidx.appcompat:appcompat:1.7.0'
-    implementation 'androidx.core:core-ktx:1.13.1'
+    implementation 'androidx.core:core-ktx:1.15.0'
     implementation 'com.google.android.material:material:1.12.0'
-    implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
+    implementation 'androidx.constraintlayout:constraintlayout:2.2.0'
     implementation 'androidx.lifecycle:lifecycle-extensions:2.2.0'
-    def androidXNavigationVersion = '2.8.2'
-    implementation "androidx.navigation:navigation-fragment-ktx:$androidXNavigationVersion"
-    implementation "androidx.navigation:navigation-ui-ktx:$androidXNavigationVersion"
-    implementation "androidx.navigation:navigation-fragment-ktx:$androidXNavigationVersion"
-    implementation "androidx.navigation:navigation-ui-ktx:$androidXNavigationVersion"
-    api 'com.github.AppDevNext.Logcat:LogcatCoreLib:3.2'
-    api 'com.github.AppDevNext.Logcat:LogcatCoreUI:3.2'
+
+    def androidx_navigation_version = '2.8.3'
+    implementation "androidx.navigation:navigation-fragment-ktx:$androidx_navigation_version"
+    implementation "androidx.navigation:navigation-ui-ktx:$androidx_navigation_version"
+    implementation "androidx.navigation:navigation-fragment-ktx:$androidx_navigation_version"
+    implementation "androidx.navigation:navigation-ui-ktx:$androidx_navigation_version"
+
+    def logcat_core_version = '3.3.1'
+    api "com.github.AppDevNext.Logcat:LogcatCoreLib:$logcat_core_version"
+    api "com.github.AppDevNext.Logcat:LogcatCoreUI:$logcat_core_version"
+
     implementation project(':library-client-rtsp')
 }
diff --git a/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt b/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt
index 8a87e66..61c49cf 100644
--- a/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt
+++ b/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt
@@ -242,6 +242,10 @@ class LiveFragment : Fragment() {
                 binding.llRtspParams.etRtspPassword.setText(it)
         }
 
+        binding.cbExperimentalRewriteSps.setOnCheckedChangeListener { _, isChecked ->
+            binding.svVideoSurface.experimentalUpdateSpsFrameWithLowLatencyParams = isChecked
+        }
+
         binding.bnRotate0.setOnClickListener {
             binding.svVideoSurface.videoRotation = 0
             binding.ivVideoImage.videoRotation = 0
diff --git a/app/src/main/res/layout/fragment_live.xml b/app/src/main/res/layout/fragment_live.xml
index 4561cca..ea22a49 100644
--- a/app/src/main/res/layout/fragment_live.xml
+++ b/app/src/main/res/layout/fragment_live.xml
@@ -17,6 +17,14 @@
             android:id="@+id/llRtspParams"
             layout="@layout/layout_rtsp_params"/>
 
+        <CheckBox
+            android:id="@+id/cbExperimentalRewriteSps"
+            android:text="Rewrite SPS frame w/ low-latency (EXPERIMENTAL)"
+            android:checked="false"
+            android:layout_margin="5dp"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content" />
+
         <Button
             android:layout_marginTop="10dp"
             android:id="@+id/bnStartStopSurface"
diff --git a/build.gradle b/build.gradle
index a856998..3703361 100644
--- a/build.gradle
+++ b/build.gradle
@@ -4,15 +4,15 @@ buildscript {
   ext.compile_sdk_version = 35
   ext.min_sdk_version = 24
   ext.target_sdk_version = 35
-  ext.project_version_code = 512
-  ext.project_version_name = '5.1.2'
+  ext.project_version_code = 520
+  ext.project_version_name = '5.2.0'
 
   repositories {
     google()
     mavenCentral()
   }
   dependencies {
-    classpath 'com.android.tools.build:gradle:8.7.1'
+    classpath 'com.android.tools.build:gradle:8.7.2'
     classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
   }
 }
diff --git a/gradle.properties b/gradle.properties
index 90e7221..5370f74 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,3 +1,2 @@
 org.gradle.jvmargs=-Xmx1g
-android.enableJetifier=true
 android.useAndroidX=true
diff --git a/library-client-rtsp/build.gradle b/library-client-rtsp/build.gradle
index f69c2c5..b7359c0 100644
--- a/library-client-rtsp/build.gradle
+++ b/library-client-rtsp/build.gradle
@@ -38,7 +38,8 @@ android {
 }
 
 dependencies {
-    implementation 'androidx.annotation:annotation:1.9.0'
+    implementation 'androidx.annotation:annotation:1.9.1'
     implementation 'androidx.media3:media3-exoplayer:1.4.1'
-    implementation 'androidx.camera:camera-core:1.4.0-rc03'
+    implementation 'androidx.camera:camera-core:1.4.0' // YUV -> BMP conversion
+    implementation 'org.jcodec:jcodec:0.2.5' // SPS frame modification
 }
diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/RtspClient.java b/library-client-rtsp/src/main/java/com/alexvas/rtsp/RtspClient.java
index 961251c..3e92045 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/RtspClient.java
+++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/RtspClient.java
@@ -676,14 +676,14 @@ private static void readRtpData(
                         case VideoCodecUtils.NAL_SPS:
                             nalUnitSps = nalUnit;
                             // Looks like there is NAL_IDR_SLICE as well. Send it now.
-                            if (nalUnit.length > 100)
+                            if (nalUnit.length > VideoCodecUtils.MAX_NAL_SPS_SIZE)
                                 listener.onRtspVideoNalUnitReceived(nalUnit, 0, nalUnit.length, header.getTimestampMsec());
                             break;
 
                         case VideoCodecUtils.NAL_PPS:
                             nalUnitPps = nalUnit;
                             // Looks like there is NAL_IDR_SLICE as well. Send it now.
-                            if (nalUnit.length > 100)
+                            if (nalUnit.length > VideoCodecUtils.MAX_NAL_SPS_SIZE)
                                 listener.onRtspVideoNalUnitReceived(nalUnit, 0, nalUnit.length, header.getTimestampMsec());
                             break;
 
diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt
index f8e032a..c98871f 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt
+++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt
@@ -20,8 +20,11 @@ import com.alexvas.rtsp.codec.VideoDecodeThread.VideoDecoderListener
 import com.alexvas.rtsp.codec.VideoFrameQueue
 import com.alexvas.utils.NetUtils
 import com.alexvas.utils.VideoCodecUtils
+import org.jcodec.codecs.h264.io.model.SeqParameterSet
 import java.net.Socket
+import java.nio.ByteBuffer
 import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.compareTo
 import kotlin.math.min
 
 class RtspProcessor(
@@ -94,6 +97,15 @@ class RtspProcessor(
      */
     var videoDecoderType = DecoderType.HARDWARE
 
+    /**
+     * Try to modify SPS frame coming from camera with low-latency parameters to decrease video
+     * decoding latency.
+     * If SPS frame param num_ref_frames is 1 or more (e.g. for Hualai cameras), set it
+     * to 0. That should decrease decoder latency from 800 msec to 100 msec on some hardware
+     * decoders.
+     */
+    var experimentalUpdateSpsFrameWithLowLatencyParams = false
+
     /**
      * Status listener for getting RTSP event updates.
      */
@@ -192,16 +204,30 @@ class RtspProcessor(
         private var framesPerGop = 0
 
         override fun onRtspVideoNalUnitReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {
-            if (DEBUG) Log.v(TAG, "onRtspVideoNalUnitReceived(length=$length, timestamp=$timestamp)")
+            if (DEBUG) Log.v(TAG, "onRtspVideoNalUnitReceived(data.size=${data.size}, length=$length, timestamp=$timestamp)")
 
             val isH265 = videoMimeType == MediaFormat.MIMETYPE_VIDEO_HEVC
             // Search for NAL_IDR_SLICE within first 1KB maximum
             val isKeyframe = VideoCodecUtils.isAnyKeyFrame(data, offset, min(length, 1000), isH265)
+
+            var videoFrame = FrameQueue.VideoFrame(
+                VideoCodecType.H264,
+                isKeyframe,
+                data,
+                offset,
+                length,
+                timestamp,
+                capturedTimestampMs = System.currentTimeMillis()
+            )
+            if (isKeyframe && experimentalUpdateSpsFrameWithLowLatencyParams) {
+                videoFrame = getNewLowLatencyFrameFromKeyFrame(videoFrame)
+            }
+
             if (debug) {
-                val nals = ArrayList<VideoCodecUtils.NalUnit>()
-                VideoCodecUtils.getNalUnits(data, offset, length, nals, isH265)
+                nalUnitsFound.clear()
+                VideoCodecUtils.getNalUnits(videoFrame.data, videoFrame.offset, videoFrame.length, nalUnitsFound, isH265)
                 var b = StringBuilder()
-                for (nal in nals) {
+                for (nal in nalUnitsFound) {
                     b
                     .append(if (isH265)
                             VideoCodecUtils.getH265NalUnitTypeString(nal.type)
@@ -215,14 +241,14 @@ class RtspProcessor(
                 @SuppressLint("UnsafeOptInUsageError")
                 if (isKeyframe) {
                     val sps = VideoCodecUtils.getSpsNalUnitFromArray(
-                        data,
-                        offset,
+                        videoFrame.data,
+                        videoFrame.offset,
                         // Check only first 100 bytes maximum. That's enough for finding SPS NAL unit.
-                        Integer.min(length, 100),
+                        Integer.min(videoFrame.length, VideoCodecUtils.MAX_NAL_SPS_SIZE),
                         isH265
                     )
                     Log.d(TAG,
-                        "\tKey frame received ($length bytes, ts=$timestamp," +
+                        "\tKey frame received (${videoFrame.length} bytes, ts=$timestamp," +
                         " ${sps?.width}x${sps?.height}," +
                         " GoP=$framesPerGop," +
                         " profile=${sps?.profileIdc}, level=${sps?.levelIdc})")
@@ -231,18 +257,13 @@ class RtspProcessor(
                     framesPerGop++
                 }
             }
-            videoFrameQueue.push(
-                FrameQueue.VideoFrame(
-                    VideoCodecType.H264,
-                    isKeyframe,
-                    data,
-                    offset,
-                    length,
-                    timestamp,
-                    capturedTimestampMs = System.currentTimeMillis()
-                )
-            )
-            dataListener?.onRtspDataVideoNalUnitReceived(data, offset, length, timestamp)
+
+            videoFrameQueue.push(videoFrame)
+            dataListener?.onRtspDataVideoNalUnitReceived(
+                videoFrame.data,
+                videoFrame.offset,
+                videoFrame.length,
+                timestamp)
         }
 
         override fun onRtspAudioSampleReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {
@@ -439,6 +460,104 @@ class RtspProcessor(
         audioDecodeThread = null
     }
 
+// Cached values
+    private val nalUnitsFound = ArrayList<VideoCodecUtils.NalUnit>()
+    private val spsBufferReadFrame = ByteBuffer.allocate(VideoCodecUtils.MAX_NAL_SPS_SIZE)
+    private val spsBufferWriteFrame = ByteBuffer.allocate(VideoCodecUtils.MAX_NAL_SPS_SIZE)
+
+    /**
+     * Try to get a new frame keyframe (SPS+PPS+IDR) with low latency modified SPS frame.
+     * If modification failed, original frame will be returned.
+     * Inspired by https://webrtc.googlesource.com/src/+/refs/heads/main/common_video/h264/sps_vui_rewriter.cc#400
+     */
+    private fun getNewLowLatencyFrameFromKeyFrame(frame: FrameQueue.VideoFrame): FrameQueue.VideoFrame {
+        try {
+            // Support only H264 for now
+            if (frame.codecType == VideoCodecType.H265)
+                return frame
+
+            nalUnitsFound.clear()
+            VideoCodecUtils.getNalUnits(frame.data, frame.offset, frame.length, nalUnitsFound, isH265 = false)
+
+            val oldSpsNalUnit = nalUnitsFound.firstOrNull { it.type == VideoCodecUtils.NAL_SPS }
+
+            // SPS frame not found. Return original frame.
+            if (oldSpsNalUnit == null)
+                return frame
+
+            spsBufferReadFrame.apply {
+                rewind()
+                put(frame.data, oldSpsNalUnit.offset + 5,
+                    Integer.min(oldSpsNalUnit.length, VideoCodecUtils.MAX_NAL_SPS_SIZE)
+                )
+                rewind()
+            }
+            // Read SPS frame
+            val spsSet = SeqParameterSet.read(spsBufferReadFrame)
+
+            // SPS frame already has num_ref_frames set to 0. Return original frame.
+            if (spsSet.numRefFrames == 0)
+                return frame
+
+            // Change SPS frame
+            if (debug)
+                Log.d(TAG, "Changed SPS num_ref_frames ${spsSet.numRefFrames} -> 0")
+            spsSet.numRefFrames = 0
+
+            // Write SPS frame
+            spsBufferWriteFrame.rewind()
+            spsSet.write(spsBufferWriteFrame)
+
+            val newSpsNalUnitSize = spsBufferWriteFrame.position()
+
+            if (oldSpsNalUnit.length > -1) {
+                val newSize = frame.length - oldSpsNalUnit.length + newSpsNalUnitSize
+                val newData = ByteArray(newSize + 5)
+                var newDataOffset = 0
+
+                for (nalUnit in nalUnitsFound) {
+                    when (nalUnit.type) {
+                        VideoCodecUtils.NAL_SPS -> {
+                            // Write NAL header + SPS frame type
+                            val b = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x27)
+                            b.copyInto(newData, newDataOffset, 0, b.size)
+                            newDataOffset += b.size
+                            // Write SPS frame body
+                            spsBufferWriteFrame.apply {
+                                rewind()
+                                get(newData, newDataOffset, newSpsNalUnitSize)
+                            }
+                            newDataOffset += newSpsNalUnitSize
+                        }
+
+                        else -> {
+                            frame.data.copyInto(
+                                newData,
+                                newDataOffset,
+                                nalUnit.offset,
+                                nalUnit.offset + nalUnit.length
+                            )
+                            newDataOffset += nalUnit.length
+                        }
+                    }
+                }
+                // Create SPS+PPS+IDR frame with newly modified SPS frame data
+                return FrameQueue.VideoFrame(
+                    frame.codecType,
+                    frame.isKeyframe,
+                    newData,
+                    0,
+                    newData.size,
+                    frame.timestampMs,
+                    frame.capturedTimestampMs
+                )
+            }
+        } catch (e: Exception) {
+            Log.e(TAG, "Failed to create low-latency keyframe", e)
+        }
+        return frame
+    }
+
     private fun getUriName(): String {
         val port = if (uri.port == -1) DEFAULT_RTSP_PORT else uri.port
         return "${uri.host.toString()}:$port"
diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt
index ad5fb59..bce6bfd 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt
+++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspSurfaceView.kt
@@ -49,6 +49,10 @@ open class RtspSurfaceView: SurfaceView {
         get() = rtspProcessor.videoDecoderType
         set(value) { rtspProcessor.videoDecoderType = value }
 
+    var experimentalUpdateSpsFrameWithLowLatencyParams: Boolean
+        get() = rtspProcessor.experimentalUpdateSpsFrameWithLowLatencyParams
+        set(value) { rtspProcessor.experimentalUpdateSpsFrameWithLowLatencyParams = value }
+
     var debug: Boolean
         get() = rtspProcessor.debug
         set(value) { rtspProcessor.debug = value }
diff --git a/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt b/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt
index 41d496c..a08e357 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt
+++ b/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt
@@ -12,6 +12,9 @@ object VideoCodecUtils {
 
     private val TAG = VideoCodecUtils::class.java.simpleName
 
+    /** Max possible NAL SPS size in bytes */
+    const val MAX_NAL_SPS_SIZE:     Int = 500
+
     const val NAL_SLICE:            Byte = 1
     const val NAL_DPA:              Byte = 2
     const val NAL_DPB:              Byte = 3