Skip to content

Commit

Permalink
Added onRtspFrameSizeChanged()
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexeyvasilyev
alexeyvasilyev committed Nov 16, 2024
1 parent edbc8d6 commit e29afdc
Show file tree
Hide file tree
Showing 7 changed files with 153 additions and 65 deletions.
66 changes: 48 additions & 18 deletions app/src/main/java/com/alexvas/rtsp/demo/live/LiveFragment.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import android.os.HandlerThread
import android.util.Log
import android.view.*
import android.widget.Toast
import androidx.constraintlayout.widget.ConstraintSet
import androidx.fragment.app.Fragment
import androidx.lifecycle.ViewModelProvider
import com.alexvas.rtsp.codec.VideoDecodeThread
Expand All @@ -29,6 +30,7 @@ class LiveFragment : Fragment() {
private lateinit var liveViewModel: LiveViewModel

private var statisticsTimer: Timer? = null
private var svVideoSurfaceResolution = Pair(0, 0)

private val rtspStatusSurfaceListener = object: RtspStatusListener {
override fun onRtspStatusConnecting() {
Expand All @@ -37,13 +39,15 @@ class LiveFragment : Fragment() {
tvStatusSurface.text = "RTSP connecting"
pbLoadingSurface.visibility = View.VISIBLE
vShutterSurface.visibility = View.VISIBLE
llRtspParams.etRtspRequest.isEnabled = false
llRtspParams.etRtspUsername.isEnabled = false
llRtspParams.etRtspPassword.isEnabled = false
llRtspParams.cbVideo.isEnabled = false
llRtspParams.cbAudio.isEnabled = false
llRtspParams.cbApplication.isEnabled = false
llRtspParams.cbDebug.isEnabled = false
llRtspParams.apply {
etRtspRequest.isEnabled = false
etRtspUsername.isEnabled = false
etRtspPassword.isEnabled = false
cbVideo.isEnabled = false
cbAudio.isEnabled = false
cbApplication.isEnabled = false
cbDebug.isEnabled = false
}
tgRotation.isEnabled = false
}
}
Expand All @@ -53,7 +57,6 @@ class LiveFragment : Fragment() {
binding.apply {
tvStatusSurface.text = "RTSP connected"
bnStartStopSurface.text = "Stop RTSP"
pbLoadingSurface.visibility = View.GONE
}
setKeepScreenOn(true)
}
Expand All @@ -73,13 +76,15 @@ class LiveFragment : Fragment() {
pbLoadingSurface.visibility = View.GONE
vShutterSurface.visibility = View.VISIBLE
pbLoadingSurface.isEnabled = false
llRtspParams.cbVideo.isEnabled = true
llRtspParams.cbAudio.isEnabled = true
llRtspParams.cbApplication.isEnabled = true
llRtspParams.cbDebug.isEnabled = true
llRtspParams.etRtspRequest.isEnabled = true
llRtspParams.etRtspUsername.isEnabled = true
llRtspParams.etRtspPassword.isEnabled = true
llRtspParams.apply {
cbVideo.isEnabled = true
cbAudio.isEnabled = true
cbApplication.isEnabled = true
cbDebug.isEnabled = true
etRtspRequest.isEnabled = true
etRtspUsername.isEnabled = true
etRtspPassword.isEnabled = true
}
tgRotation.isEnabled = true
}
setKeepScreenOn(false)
Expand Down Expand Up @@ -107,11 +112,24 @@ class LiveFragment : Fragment() {

override fun onRtspFirstFrameRendered() {
if (DEBUG) Log.v(TAG, "onRtspFirstFrameRendered()")
Log.i(TAG, "First frame rendered")
binding.apply {
pbLoadingSurface.visibility = View.GONE
vShutterSurface.visibility = View.GONE
bnSnapshotSurface.isEnabled = true
}
}

override fun onRtspFrameSizeChanged(width: Int, height: Int) {
if (DEBUG) Log.v(TAG, "onRtspFrameSizeChanged(width=$width, height=$height)")
Log.i(TAG, "Video resolution changed to ${width}x${height}")
svVideoSurfaceResolution = Pair(width, height)
ConstraintSet().apply {
clone(binding.csVideoSurface)
setDimensionRatio(binding.svVideoSurface.id, "$width:$height")
applyTo(binding.csVideoSurface)
}
}
}

private val rtspDataListener = object: RtspDataListener {
Expand All @@ -136,7 +154,6 @@ class LiveFragment : Fragment() {
binding.apply {
tvStatusImage.text = "RTSP connected"
bnStartStopImage.text = "Stop RTSP"
pbLoadingImage.visibility = View.GONE
}
setKeepScreenOn(true)
}
Expand Down Expand Up @@ -182,8 +199,20 @@ class LiveFragment : Fragment() {

override fun onRtspFirstFrameRendered() {
if (DEBUG) Log.v(TAG, "onRtspFirstFrameRendered()")
Log.i(TAG, "First frame rendered")
binding.apply {
vShutterImage.visibility = View.GONE
pbLoadingImage.visibility = View.GONE
}
}

override fun onRtspFrameSizeChanged(width: Int, height: Int) {
if (DEBUG) Log.v(TAG, "onRtspFrameSizeChanged(width=$width, height=$height)")
Log.i(TAG, "Video resolution changed to ${width}x${height}")
ConstraintSet().apply {
clone(binding.csVideoImage)
setDimensionRatio(binding.ivVideoImage.id, "$width:$height")
applyTo(binding.csVideoImage)
}
}
}
Expand Down Expand Up @@ -325,7 +354,7 @@ class LiveFragment : Fragment() {
}
}

binding.pbLoadingSurface.setOnClickListener {
binding.bnSnapshotSurface.setOnClickListener {
val bitmap = getSnapshot()
// TODO Save snapshot to DCIM folder
if (bitmap != null) {
Expand Down Expand Up @@ -364,7 +393,8 @@ class LiveFragment : Fragment() {
val statistics = binding.svVideoSurface.statistics
val text =
"Video decoder: ${statistics.videoDecoderType.toString().lowercase()} ${if (statistics.videoDecoderName.isNullOrEmpty()) "" else "(${statistics.videoDecoderName})"}" +
"\nVideo decoder latency: ${statistics.videoDecoderLatencyMsec} ms"
"\nVideo decoder latency: ${statistics.videoDecoderLatencyMsec} ms" +
"\nResolution: ${svVideoSurfaceResolution.first}x${svVideoSurfaceResolution.second}"
// "\nNetwork latency: "

// // Assume that difference between current Android time and camera time cannot be more than 5 sec.
Expand Down
97 changes: 62 additions & 35 deletions app/src/main/res/layout/fragment_live.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,18 @@

<CheckBox
android:id="@+id/cbExperimentalRewriteSps"
android:text="Rewrite SPS frame w/ low-latency (EXPERIMENTAL)"
android:text="Rewrite SPS frames w/ low-latency params (EXPERIMENTAL)"
android:checked="false"
android:layout_margin="5dp"
android:layout_width="wrap_content"
android:layout_height="wrap_content" />

<Button
android:layout_marginTop="10dp"
android:layout_marginTop="40dp"
android:id="@+id/bnStartStopSurface"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="center"
android:layout_marginBottom="10dp"
android:text="Start" />


Expand All @@ -42,32 +41,55 @@
android:layout_height="match_parent"
android:paddingBottom="5dp"
android:text="RtspSurfaceView:"/>
<FrameLayout

<androidx.constraintlayout.widget.ConstraintLayout
android:id="@+id/csVideoSurface"
android:layout_width="match_parent"
android:layout_height="300dp"
android:layout_height="match_parent"
android:animateLayoutChanges="true">
<com.alexvas.rtsp.widget.RtspSurfaceView
android:layout_width="match_parent"
android:layout_height="match_parent"
android:id="@+id/svVideoSurface" />
android:layout_width="0dp"
android:layout_height="0dp"
android:id="@+id/svVideoSurface"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintDimensionRatio="16:9"/>
<View
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_width="0dp"
android:layout_height="0dp"
android:background="@android:color/black"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:id="@+id/vShutterSurface" />
<ProgressBar
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="center"
android:visibility="gone"
android:id="@+id/pbLoadingSurface"/>
</FrameLayout>
android:id="@+id/pbLoadingSurface"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"/>
</androidx.constraintlayout.widget.ConstraintLayout>

<!-- Debug statistics -->
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="2dp"
android:id="@+id/tvStatistics"
android:textSize="12sp"/>

<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:padding="5dp"
android:paddingStart="5dp"
android:paddingEnd="5dp"
android:orientation="horizontal"
android:gravity="center_vertical">
<Button
Expand All @@ -76,7 +98,7 @@
style="@style/Widget.Material3.Button.TextButton.Icon"
android:id="@+id/bnSnapshotSurface"
android:enabled="false"
android:text="Photo"
android:text="Snapshot"
app:icon="@drawable/ic_camera_black_24dp"/>
<TextView
android:layout_width="match_parent"
Expand All @@ -89,40 +111,54 @@
<!-- RtspImageView -->

<Button
android:layout_marginTop="10dp"
android:layout_marginTop="30dp"
android:id="@+id/bnStartStopImage"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="center"
android:layout_marginBottom="10dp"
android:text="Start" />

<TextView
android:layout_width="wrap_content"
android:layout_height="match_parent"
android:paddingBottom="5dp"
android:text="RtspImageView:"/>
<FrameLayout

<androidx.constraintlayout.widget.ConstraintLayout
android:id="@+id/csVideoImage"
android:layout_width="match_parent"
android:layout_height="300dp"
android:layout_height="match_parent"
android:animateLayoutChanges="true">
<com.alexvas.rtsp.widget.RtspImageView
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_width="0dp"
android:layout_height="0dp"
android:scaleType="fitXY"
android:id="@+id/ivVideoImage" />
android:id="@+id/ivVideoImage"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintDimensionRatio="16:9"/>
<View
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_width="0dp"
android:layout_height="0dp"
android:background="@android:color/black"
android:id="@+id/vShutterImage" />
android:id="@+id/vShutterImage"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<ProgressBar
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="center"
android:visibility="gone"
android:id="@+id/pbLoadingImage"/>
</FrameLayout>
android:id="@+id/pbLoadingImage"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout>

<LinearLayout
android:layout_width="match_parent"
Expand Down Expand Up @@ -221,15 +257,6 @@
</com.google.android.material.button.MaterialButtonToggleGroup>
</LinearLayout>

<!-- Debug statistics -->
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="10dp"
android:layout_margin="6dp"
android:id="@+id/tvStatistics"
android:textSize="12sp"/>

</LinearLayout>

</ScrollView>
4 changes: 2 additions & 2 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ buildscript {
ext.compile_sdk_version = 35
ext.min_sdk_version = 24
ext.target_sdk_version = 35
ext.project_version_code = 521
ext.project_version_name = '5.2.1'
ext.project_version_code = 530
ext.project_version_name = '5.3.0'

repositories {
google()
Expand Down
33 changes: 31 additions & 2 deletions library-client-rtsp/src/main/java/com/alexvas/rtsp/codec/VideoDecodeThread.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ import android.util.Log
import com.alexvas.utils.MediaCodecUtils
import com.alexvas.utils.capabilitiesToString
import androidx.media3.common.util.Util
import com.alexvas.utils.VideoCodecUtils
import com.limelight.binding.video.MediaCodecHelper
import java.lang.Integer.min
import java.nio.ByteBuffer
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
Expand Down Expand Up @@ -276,6 +278,8 @@ abstract class VideoDecodeThread (
val bufferInfo = MediaCodec.BufferInfo()

try {
var widthHeightFromStream: Pair<Int, Int>? = null

// Map for calculating decoder rendering latency.
// key - original frame timestamp, value - timestamp when frame was added to the map
val keyframesTimestamps = HashMap<Long, Long>()
Expand Down Expand Up @@ -323,7 +327,26 @@ abstract class VideoDecodeThread (
Log.i(TAG, "\tFrame queued (${l - frameQueuedMsec}) ${if (frame.isKeyframe) "key frame" else ""}")
frameQueuedMsec = l
}
decoder.queueInputBuffer(inIndex, frame.offset, frame.length, frame.timestampMs, 0)
val flags = if (frame.isKeyframe)
(MediaCodec.BUFFER_FLAG_KEY_FRAME /*or MediaCodec.BUFFER_FLAG_CODEC_CONFIG*/) else 0
decoder.queueInputBuffer(inIndex, frame.offset, frame.length, frame.timestampMs, flags)

if (frame.isKeyframe) {
// Obtain width and height from stream
widthHeightFromStream = try {
VideoCodecUtils.getWidthHeightFromArray(
frame.data,
frame.offset,
// Check only first 100 bytes maximum. That's enough for finding SPS NAL unit.
min(frame.length, VideoCodecUtils.MAX_NAL_SPS_SIZE),
isH265 = frame.codecType == VideoCodecType.H265
)
} catch (_: Exception) {
// Log.e(TAG, "Failed to parse width/height from SPS frame. SPS frame seems to be corrupted.", e)
null
}
// Log.i(TAG, "width/height: ${widthHeightFromStream?.first}x${widthHeightFromStream?.second}")
}
}
}

Expand All @@ -341,7 +364,13 @@ abstract class VideoDecodeThread (
// Resolution changed
MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED, MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
Log.d(TAG, "Decoder format changed: ${decoder.outputFormat}")
val widthHeight = getWidthHeight(decoder.outputFormat)
// Decoder can contain different resolution (it can make downsampling).
// If resolution successfully obtained from SPS frame, use it.
val widthHeightFromDecoder = getWidthHeight(decoder.outputFormat)
val widthHeight = widthHeightFromStream ?: widthHeightFromDecoder
Log.i(TAG, "Video decoder resolution: ${widthHeightFromDecoder.first}x${widthHeightFromDecoder.second}, stream resolution: ${widthHeightFromStream?.first}x${widthHeightFromStream?.second}")

// val widthHeightFromDecoder = getWidthHeight(decoder.outputFormat)
val rotation = if (decoder.outputFormat.containsKey(MediaFormat.KEY_ROTATION)) {
decoder.outputFormat.getInteger(MediaFormat.KEY_ROTATION)
} else {
Expand Down
Loading

0 comments on commit e29afdc

Please sign in to comment.