From e29afdc15f06069773f78281a9d8342650e3972a Mon Sep 17 00:00:00 2001
From: Alexey Vasilyev <alexey.vasilyev@gmail.com>
Date: Sat, 16 Nov 2024 19:20:28 +0200
Subject: [PATCH] Added onRtspFrameSizeChanged()

---
 .../alexvas/rtsp/demo/live/LiveFragment.kt    | 66 +++++++++----
 app/src/main/res/layout/fragment_live.xml     | 97 ++++++++++++-------
 build.gradle                                  |  4 +-
 .../alexvas/rtsp/codec/VideoDecodeThread.kt   | 33 ++++++-
 .../com/alexvas/rtsp/widget/RtspListeners.kt  |  1 +
 .../com/alexvas/rtsp/widget/RtspProcessor.kt  |  1 +
 .../java/com/alexvas/utils/VideoCodecUtils.kt | 16 +--
 7 files changed, 153 insertions(+), 65 deletions(-)

diff --git a/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt b/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt
index 61c49cf..b39ef32 100644
--- a/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt
+++ b/app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt
@@ -9,6 +9,7 @@ import android.os.HandlerThread
 import android.util.Log
 import android.view.*
 import android.widget.Toast
+import androidx.constraintlayout.widget.ConstraintSet
 import androidx.fragment.app.Fragment
 import androidx.lifecycle.ViewModelProvider
 import com.alexvas.rtsp.codec.VideoDecodeThread
@@ -29,6 +30,7 @@ class LiveFragment : Fragment() {
     private lateinit var liveViewModel: LiveViewModel
 
     private var statisticsTimer: Timer? = null
+    private var svVideoSurfaceResolution = Pair(0, 0)
 
     private val rtspStatusSurfaceListener = object: RtspStatusListener {
         override fun onRtspStatusConnecting() {
@@ -37,13 +39,15 @@ class LiveFragment : Fragment() {
                 tvStatusSurface.text = "RTSP connecting"
                 pbLoadingSurface.visibility = View.VISIBLE
                 vShutterSurface.visibility = View.VISIBLE
-                llRtspParams.etRtspRequest.isEnabled = false
-                llRtspParams.etRtspUsername.isEnabled = false
-                llRtspParams.etRtspPassword.isEnabled = false
-                llRtspParams.cbVideo.isEnabled = false
-                llRtspParams.cbAudio.isEnabled = false
-                llRtspParams.cbApplication.isEnabled = false
-                llRtspParams.cbDebug.isEnabled = false
+                llRtspParams.apply {
+                    etRtspRequest.isEnabled = false
+                    etRtspUsername.isEnabled = false
+                    etRtspPassword.isEnabled = false
+                    cbVideo.isEnabled = false
+                    cbAudio.isEnabled = false
+                    cbApplication.isEnabled = false
+                    cbDebug.isEnabled = false
+                }
                 tgRotation.isEnabled = false
             }
         }
@@ -53,7 +57,6 @@ class LiveFragment : Fragment() {
             binding.apply {
                 tvStatusSurface.text = "RTSP connected"
                 bnStartStopSurface.text = "Stop RTSP"
-                pbLoadingSurface.visibility = View.GONE
             }
             setKeepScreenOn(true)
         }
@@ -73,13 +76,15 @@ class LiveFragment : Fragment() {
                 pbLoadingSurface.visibility = View.GONE
                 vShutterSurface.visibility = View.VISIBLE
                 pbLoadingSurface.isEnabled = false
-                llRtspParams.cbVideo.isEnabled = true
-                llRtspParams.cbAudio.isEnabled = true
-                llRtspParams.cbApplication.isEnabled = true
-                llRtspParams.cbDebug.isEnabled = true
-                llRtspParams.etRtspRequest.isEnabled = true
-                llRtspParams.etRtspUsername.isEnabled = true
-                llRtspParams.etRtspPassword.isEnabled = true
+                llRtspParams.apply {
+                    cbVideo.isEnabled = true
+                    cbAudio.isEnabled = true
+                    cbApplication.isEnabled = true
+                    cbDebug.isEnabled = true
+                    etRtspRequest.isEnabled = true
+                    etRtspUsername.isEnabled = true
+                    etRtspPassword.isEnabled = true
+                }
                 tgRotation.isEnabled = true
             }
             setKeepScreenOn(false)
@@ -107,11 +112,24 @@ class LiveFragment : Fragment() {
 
         override fun onRtspFirstFrameRendered() {
             if (DEBUG) Log.v(TAG, "onRtspFirstFrameRendered()")
+            Log.i(TAG, "First frame rendered")
             binding.apply {
+                pbLoadingSurface.visibility = View.GONE
                 vShutterSurface.visibility = View.GONE
                 bnSnapshotSurface.isEnabled = true
             }
         }
+
+        override fun onRtspFrameSizeChanged(width: Int, height: Int) {
+            if (DEBUG) Log.v(TAG, "onRtspFrameSizeChanged(width=$width, height=$height)")
+            Log.i(TAG, "Video resolution changed to ${width}x${height}")
+            svVideoSurfaceResolution = Pair(width, height)
+            ConstraintSet().apply {
+                clone(binding.csVideoSurface)
+                setDimensionRatio(binding.svVideoSurface.id, "$width:$height")
+                applyTo(binding.csVideoSurface)
+            }
+        }
     }
 
     private val rtspDataListener = object: RtspDataListener {
@@ -136,7 +154,6 @@ class LiveFragment : Fragment() {
             binding.apply {
                 tvStatusImage.text = "RTSP connected"
                 bnStartStopImage.text = "Stop RTSP"
-                pbLoadingImage.visibility = View.GONE
             }
             setKeepScreenOn(true)
         }
@@ -182,8 +199,20 @@ class LiveFragment : Fragment() {
 
         override fun onRtspFirstFrameRendered() {
             if (DEBUG) Log.v(TAG, "onRtspFirstFrameRendered()")
+            Log.i(TAG, "First frame rendered")
             binding.apply {
                 vShutterImage.visibility = View.GONE
+                pbLoadingImage.visibility = View.GONE
+            }
+        }
+
+        override fun onRtspFrameSizeChanged(width: Int, height: Int) {
+            if (DEBUG) Log.v(TAG, "onRtspFrameSizeChanged(width=$width, height=$height)")
+            Log.i(TAG, "Video resolution changed to ${width}x${height}")
+            ConstraintSet().apply {
+                clone(binding.csVideoImage)
+                setDimensionRatio(binding.ivVideoImage.id, "$width:$height")
+                applyTo(binding.csVideoImage)
             }
         }
     }
@@ -325,7 +354,7 @@ class LiveFragment : Fragment() {
             }
         }
 
-        binding.pbLoadingSurface.setOnClickListener {
+        binding.bnSnapshotSurface.setOnClickListener {
             val bitmap = getSnapshot()
             // TODO Save snapshot to DCIM folder
             if (bitmap != null) {
@@ -364,7 +393,8 @@ class LiveFragment : Fragment() {
                     val statistics = binding.svVideoSurface.statistics
                     val text =
                         "Video decoder: ${statistics.videoDecoderType.toString().lowercase()} ${if (statistics.videoDecoderName.isNullOrEmpty()) "" else "(${statistics.videoDecoderName})"}" +
-                        "\nVideo decoder latency: ${statistics.videoDecoderLatencyMsec} ms"
+                        "\nVideo decoder latency: ${statistics.videoDecoderLatencyMsec} ms" +
+                        "\nResolution: ${svVideoSurfaceResolution.first}x${svVideoSurfaceResolution.second}"
 //                        "\nNetwork latency: "
 
 //                    // Assume that difference between current Android time and camera time cannot be more than 5 sec.
diff --git a/app/src/main/res/layout/fragment_live.xml b/app/src/main/res/layout/fragment_live.xml
index ea22a49..fc5a60d 100644
--- a/app/src/main/res/layout/fragment_live.xml
+++ b/app/src/main/res/layout/fragment_live.xml
@@ -19,19 +19,18 @@
 
         <CheckBox
             android:id="@+id/cbExperimentalRewriteSps"
-            android:text="Rewrite SPS frame w/ low-latency (EXPERIMENTAL)"
+            android:text="Rewrite SPS frames w/ low-latency params (EXPERIMENTAL)"
             android:checked="false"
             android:layout_margin="5dp"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content" />
 
         <Button
-            android:layout_marginTop="10dp"
+            android:layout_marginTop="40dp"
             android:id="@+id/bnStartStopSurface"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
             android:layout_gravity="center"
-            android:layout_marginBottom="10dp"
             android:text="Start" />
 
 
@@ -42,32 +41,55 @@
             android:layout_height="match_parent"
             android:paddingBottom="5dp"
             android:text="RtspSurfaceView:"/>
-        <FrameLayout
+
+        <androidx.constraintlayout.widget.ConstraintLayout
+            android:id="@+id/csVideoSurface"
             android:layout_width="match_parent"
-            android:layout_height="300dp"
+            android:layout_height="match_parent"
             android:animateLayoutChanges="true">
             <com.alexvas.rtsp.widget.RtspSurfaceView
-                android:layout_width="match_parent"
-                android:layout_height="match_parent"
-                android:id="@+id/svVideoSurface" />
+                android:layout_width="0dp"
+                android:layout_height="0dp"
+                android:id="@+id/svVideoSurface"
+                app:layout_constraintTop_toTopOf="parent"
+                app:layout_constraintBottom_toBottomOf="parent"
+                app:layout_constraintStart_toStartOf="parent"
+                app:layout_constraintEnd_toEndOf="parent"
+                app:layout_constraintDimensionRatio="16:9"/>
             <View
-                android:layout_width="match_parent"
-                android:layout_height="match_parent"
+                android:layout_width="0dp"
+                android:layout_height="0dp"
                 android:background="@android:color/black"
+                app:layout_constraintTop_toTopOf="parent"
+                app:layout_constraintBottom_toBottomOf="parent"
+                app:layout_constraintStart_toStartOf="parent"
+                app:layout_constraintEnd_toEndOf="parent"
                 android:id="@+id/vShutterSurface" />
             <ProgressBar
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
                 android:layout_gravity="center"
                 android:visibility="gone"
-                android:id="@+id/pbLoadingSurface"/>
-        </FrameLayout>
+                android:id="@+id/pbLoadingSurface"
+                app:layout_constraintTop_toTopOf="parent"
+                app:layout_constraintBottom_toBottomOf="parent"
+                app:layout_constraintStart_toStartOf="parent"
+                app:layout_constraintEnd_toEndOf="parent"/>
+        </androidx.constraintlayout.widget.ConstraintLayout>
 
+        <!-- Debug statistics -->
+        <TextView
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_marginTop="2dp"
+            android:id="@+id/tvStatistics"
+            android:textSize="12sp"/>
 
         <LinearLayout
             android:layout_width="match_parent"
             android:layout_height="wrap_content"
-            android:padding="5dp"
+            android:paddingStart="5dp"
+            android:paddingEnd="5dp"
             android:orientation="horizontal"
             android:gravity="center_vertical">
             <Button
@@ -76,7 +98,7 @@
                 style="@style/Widget.Material3.Button.TextButton.Icon"
                 android:id="@+id/bnSnapshotSurface"
                 android:enabled="false"
-                android:text="Photo"
+                android:text="Snapshot"
                 app:icon="@drawable/ic_camera_black_24dp"/>
             <TextView
                 android:layout_width="match_parent"
@@ -89,12 +111,11 @@
         <!-- RtspImageView -->
 
         <Button
-            android:layout_marginTop="10dp"
+            android:layout_marginTop="30dp"
             android:id="@+id/bnStartStopImage"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
             android:layout_gravity="center"
-            android:layout_marginBottom="10dp"
             android:text="Start" />
 
         <TextView
@@ -102,27 +123,42 @@
             android:layout_height="match_parent"
             android:paddingBottom="5dp"
             android:text="RtspImageView:"/>
-        <FrameLayout
+
+        <androidx.constraintlayout.widget.ConstraintLayout
+            android:id="@+id/csVideoImage"
             android:layout_width="match_parent"
-            android:layout_height="300dp"
+            android:layout_height="match_parent"
             android:animateLayoutChanges="true">
             <com.alexvas.rtsp.widget.RtspImageView
-                android:layout_width="match_parent"
-                android:layout_height="match_parent"
+                android:layout_width="0dp"
+                android:layout_height="0dp"
                 android:scaleType="fitXY"
-                android:id="@+id/ivVideoImage" />
+                android:id="@+id/ivVideoImage"
+                app:layout_constraintTop_toTopOf="parent"
+                app:layout_constraintBottom_toBottomOf="parent"
+                app:layout_constraintStart_toStartOf="parent"
+                app:layout_constraintEnd_toEndOf="parent"
+                app:layout_constraintDimensionRatio="16:9"/>
             <View
-                android:layout_width="match_parent"
-                android:layout_height="match_parent"
+                android:layout_width="0dp"
+                android:layout_height="0dp"
                 android:background="@android:color/black"
-                android:id="@+id/vShutterImage" />
+                android:id="@+id/vShutterImage"
+                app:layout_constraintTop_toTopOf="parent"
+                app:layout_constraintBottom_toBottomOf="parent"
+                app:layout_constraintStart_toStartOf="parent"
+                app:layout_constraintEnd_toEndOf="parent" />
             <ProgressBar
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
                 android:layout_gravity="center"
                 android:visibility="gone"
-                android:id="@+id/pbLoadingImage"/>
-        </FrameLayout>
+                android:id="@+id/pbLoadingImage"
+                app:layout_constraintTop_toTopOf="parent"
+                app:layout_constraintBottom_toBottomOf="parent"
+                app:layout_constraintStart_toStartOf="parent"
+                app:layout_constraintEnd_toEndOf="parent" />
+        </androidx.constraintlayout.widget.ConstraintLayout>
 
         <LinearLayout
             android:layout_width="match_parent"
@@ -221,15 +257,6 @@
             </com.google.android.material.button.MaterialButtonToggleGroup>
         </LinearLayout>
 
-        <!-- Debug statistics -->
-        <TextView
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:layout_marginTop="10dp"
-            android:layout_margin="6dp"
-            android:id="@+id/tvStatistics"
-            android:textSize="12sp"/>
-
     </LinearLayout>
 
 </ScrollView>
\ No newline at end of file
diff --git a/build.gradle b/build.gradle
index 2c2c570..aac98d7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -4,8 +4,8 @@ buildscript {
   ext.compile_sdk_version = 35
   ext.min_sdk_version = 24
   ext.target_sdk_version = 35
-  ext.project_version_code = 521
-  ext.project_version_name = '5.2.1'
+  ext.project_version_code = 530
+  ext.project_version_name = '5.3.0'
 
   repositories {
     google()
diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt
index 1ddbbf6..0d6a244 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt
+++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt
@@ -12,7 +12,9 @@ import android.util.Log
 import com.alexvas.utils.MediaCodecUtils
 import com.alexvas.utils.capabilitiesToString
 import androidx.media3.common.util.Util
+import com.alexvas.utils.VideoCodecUtils
 import com.limelight.binding.video.MediaCodecHelper
+import java.lang.Integer.min
 import java.nio.ByteBuffer
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicBoolean
@@ -276,6 +278,8 @@ abstract class VideoDecodeThread (
             val bufferInfo = MediaCodec.BufferInfo()
 
             try {
+                var widthHeightFromStream: Pair<Int, Int>? = null
+
                 // Map for calculating decoder rendering latency.
                 // key - original frame timestamp, value - timestamp when frame was added to the map
                 val keyframesTimestamps = HashMap<Long, Long>()
@@ -323,7 +327,26 @@ abstract class VideoDecodeThread (
                                     Log.i(TAG, "\tFrame queued (${l - frameQueuedMsec}) ${if (frame.isKeyframe) "key frame" else ""}")
                                     frameQueuedMsec = l
                                 }
-                                decoder.queueInputBuffer(inIndex, frame.offset, frame.length, frame.timestampMs, 0)
+                                val flags = if (frame.isKeyframe)
+                                    (MediaCodec.BUFFER_FLAG_KEY_FRAME /*or MediaCodec.BUFFER_FLAG_CODEC_CONFIG*/) else 0
+                                decoder.queueInputBuffer(inIndex, frame.offset, frame.length, frame.timestampMs, flags)
+
+                                if (frame.isKeyframe) {
+                                    // Obtain width and height from stream
+                                    widthHeightFromStream = try {
+                                        VideoCodecUtils.getWidthHeightFromArray(
+                                            frame.data,
+                                            frame.offset,
+                                            // Check only first 100 bytes maximum. That's enough for finding SPS NAL unit.
+                                            min(frame.length, VideoCodecUtils.MAX_NAL_SPS_SIZE),
+                                            isH265 = frame.codecType == VideoCodecType.H265
+                                        )
+                                    } catch (_: Exception) {
+//                                      Log.e(TAG, "Failed to parse width/height from SPS frame. SPS frame seems to be corrupted.", e)
+                                        null
+                                    }
+//                                  Log.i(TAG, "width/height: ${widthHeightFromStream?.first}x${widthHeightFromStream?.second}")
+                                }
                             }
                         }
 
@@ -341,7 +364,13 @@ abstract class VideoDecodeThread (
                                 // Resolution changed
                                 MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED, MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
                                     Log.d(TAG, "Decoder format changed: ${decoder.outputFormat}")
-                                    val widthHeight = getWidthHeight(decoder.outputFormat)
+                                    // Decoder can contain different resolution (it can make downsampling).
+                                    // If resolution successfully obtained from SPS frame, use it.
+                                    val widthHeightFromDecoder = getWidthHeight(decoder.outputFormat)
+                                    val widthHeight = widthHeightFromStream ?: widthHeightFromDecoder
+                                    Log.i(TAG, "Video decoder resolution: ${widthHeightFromDecoder.first}x${widthHeightFromDecoder.second}, stream resolution: ${widthHeightFromStream?.first}x${widthHeightFromStream?.second}")
+
+//                                    val widthHeightFromDecoder = getWidthHeight(decoder.outputFormat)
                                     val rotation = if (decoder.outputFormat.containsKey(MediaFormat.KEY_ROTATION)) {
                                         decoder.outputFormat.getInteger(MediaFormat.KEY_ROTATION)
                                     } else {
diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspListeners.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspListeners.kt
index 5881f2f..2c0b48a 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspListeners.kt
+++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspListeners.kt
@@ -11,6 +11,7 @@ interface RtspStatusListener {
     fun onRtspStatusFailedUnauthorized() {}
     fun onRtspStatusFailed(message: String?) {}
     fun onRtspFirstFrameRendered() {}
+    fun onRtspFrameSizeChanged(width: Int, height: Int) {}
 }
 
 /**
diff --git a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt
index 0a84145..6cb7770 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt
+++ b/library-client-rtsp/src/main/java/com/alexvas/rtsp/widget/RtspProcessor.kt
@@ -371,6 +371,7 @@ class RtspProcessor(
 
         override fun onVideoDecoderFormatChanged(width: Int, height: Int) {
             if (DEBUG) Log.v(TAG, "onVideoDecoderFormatChanged(width=$width, height=$height)")
+            statusListener?.onRtspFrameSizeChanged(width, height)
         }
 
         override fun onVideoDecoderFirstFrameRendered() {
diff --git a/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt b/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt
index a08e357..1ffa0ed 100644
--- a/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt
+++ b/library-client-rtsp/src/main/java/com/alexvas/utils/VideoCodecUtils.kt
@@ -274,14 +274,14 @@ object VideoCodecUtils {
         return null
     }
 
-//    @SuppressLint("UnsafeOptInUsageError")
-//    fun getWidthHeightFromArray(src: ByteArray, offset: Int, length: Int): Pair<Int, Int>? {
-//        val sps = getSpsNalUnitFromArray(src, offset, length)
-//        sps?.let {
-//            return Pair(sps.width, sps.height)
-//        }
-//        return null
-//    }
+    @SuppressLint("UnsafeOptInUsageError")
+    fun getWidthHeightFromArray(src: ByteArray, offset: Int, length: Int, isH265: Boolean): Pair<Int, Int>? {
+        val sps = getSpsNalUnitFromArray(src, offset, length, isH265)
+        sps?.let {
+            return Pair(sps.width, sps.height)
+        }
+        return null
+    }
 
 
 //    private fun isH265IRAP(nalUnitType: Byte): Boolean {