From 37f8067bac51c115a480505daeb7853c98f03033 Mon Sep 17 00:00:00 2001 From: Alex Shepeliev Date: Sat, 17 Aug 2024 19:13:09 +0300 Subject: [PATCH] Add video processor factory for Android and iOS targets (#125) --- .../kotlin/com/shepeliev/webrtckmp/MediaDevices.kt | 2 ++ .../shepeliev/webrtckmp/VideoProcessorFactory.kt | 7 +++++++ .../kotlin/com/shepeliev/webrtckmp/WebRtc.kt | 2 ++ .../shepeliev/webrtckmp/MediaTrackConstraints.kt | 4 ++-- .../webrtckmp/CameraVideoCaptureController.kt | 6 +++--- .../kotlin/com/shepeliev/webrtckmp/MediaDevices.kt | 14 ++++++++++---- .../shepeliev/webrtckmp/VideoProcessorFactory.kt | 10 ++++++++++ .../kotlin/com/shepeliev/webrtckmp/WebRtc.kt | 1 + 8 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt create mode 100644 webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt diff --git a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt index ade6cf7..f0a0f6a 100644 --- a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt +++ b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt @@ -43,6 +43,7 @@ private object MediaDevicesImpl : MediaDevices { if (constraints.video != null) { checkCameraPermission() val videoSource = WebRtc.peerConnectionFactory.createVideoSource(false) + videoSource.setVideoProcessor(WebRtc.videoProcessorFactory?.createVideoProcessor()) val videoCaptureController = CameraVideoCaptureController( constraints.video, videoSource @@ -62,6 +63,7 @@ private object MediaDevicesImpl : MediaDevices { override suspend fun getDisplayMedia(): MediaStream { val videoSource = WebRtc.peerConnectionFactory.createVideoSource(false) + WebRtc.videoProcessorFactory?.createVideoProcessor()?.let { videoSource.setVideoProcessor(it) } val screenCaptureController = ScreenCaptureController(videoSource) val videoTrack = WebRtc.peerConnectionFactory.createVideoTrack( UUID.randomUUID().toString(), diff --git a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt new file mode 100644 index 0000000..bcc002c --- /dev/null +++ b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt @@ -0,0 +1,7 @@ +package com.shepeliev.webrtckmp + +import org.webrtc.VideoProcessor + +fun interface VideoProcessorFactory { + fun createVideoProcessor(): VideoProcessor +} diff --git a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt index 498b2b4..d42f54e 100644 --- a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt +++ b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt @@ -24,6 +24,8 @@ object WebRtc { var videoDecoderFactory: VideoDecoderFactory? = null var customCameraEnumerator: CameraEnumerator? = null var customPeerConnectionFactory: PeerConnectionFactory? = null + var videoProcessorFactory: VideoProcessorFactory? = null + lateinit var factoryInitializationOptionsBuilder: PeerConnectionFactory.InitializationOptions.Builder private set diff --git a/webrtc-kmp/src/commonMain/kotlin/com/shepeliev/webrtckmp/MediaTrackConstraints.kt b/webrtc-kmp/src/commonMain/kotlin/com/shepeliev/webrtckmp/MediaTrackConstraints.kt index 4c203d3..c30e9b6 100644 --- a/webrtc-kmp/src/commonMain/kotlin/com/shepeliev/webrtckmp/MediaTrackConstraints.kt +++ b/webrtc-kmp/src/commonMain/kotlin/com/shepeliev/webrtckmp/MediaTrackConstraints.kt @@ -162,8 +162,8 @@ class MediaTrackConstraintsBuilder(internal var constraints: MediaTrackConstrain constraints = constraints.copy(facingMode = constrain) } - fun frameRate(ratio: Double) { - constraints = constraints.copy(frameRate = ratio.asValueConstrain()) + fun frameRate(fps: Double) { + constraints = constraints.copy(frameRate = fps.asValueConstrain()) } fun frameRate(build: ValueOrConstrain.Constrain.() -> Unit) { diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/CameraVideoCaptureController.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/CameraVideoCaptureController.kt index 694f00d..7db8976 100644 --- a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/CameraVideoCaptureController.kt +++ b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/CameraVideoCaptureController.kt @@ -3,7 +3,7 @@ package com.shepeliev.webrtckmp import WebRTC.RTCCameraVideoCapturer -import WebRTC.RTCVideoSource +import WebRTC.RTCVideoCapturerDelegateProtocol import kotlinx.cinterop.ExperimentalForeignApi import kotlinx.cinterop.useContents import platform.AVFoundation.AVCaptureDevice @@ -19,7 +19,7 @@ import kotlin.math.abs internal class CameraVideoCaptureController( private val constraints: MediaTrackConstraints, - private val videoSource: RTCVideoSource, + private val videoCapturerDelegate: RTCVideoCapturerDelegateProtocol, ) : VideoCaptureController { private var videoCapturer: RTCCameraVideoCapturer? = null private var position: AVCaptureDevicePosition = AVCaptureDevicePositionBack @@ -32,7 +32,7 @@ internal class CameraVideoCaptureController( override fun startCapture() { if (videoCapturer != null) return - videoCapturer = RTCCameraVideoCapturer(videoSource) + videoCapturer = RTCCameraVideoCapturer(videoCapturerDelegate) if (!this::device.isInitialized) selectDevice() selectFormat() selectFps() diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt index df9929f..955d2a3 100644 --- a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt +++ b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt @@ -1,5 +1,3 @@ -@file:OptIn(ExperimentalForeignApi::class) - package com.shepeliev.webrtckmp import WebRTC.RTCCameraVideoCapturer @@ -10,6 +8,7 @@ import platform.Foundation.NSUUID internal actual val mediaDevices: MediaDevices = MediaDevicesImpl +@OptIn(ExperimentalForeignApi::class) private object MediaDevicesImpl : MediaDevices { override suspend fun getUserMedia(streamConstraints: MediaStreamConstraintsBuilder.() -> Unit): MediaStream { val constraints = MediaStreamConstraintsBuilder().let { @@ -29,8 +28,15 @@ private object MediaDevicesImpl : MediaDevices { val videoTrack = constraints.video?.let { videoConstraints -> val videoSource = WebRtc.peerConnectionFactory.videoSource() - val iosVideoTrack = WebRtc.peerConnectionFactory.videoTrackWithSource(videoSource, NSUUID.UUID().UUIDString()) - val videoCaptureController = CameraVideoCaptureController(videoConstraints, videoSource) + val videoProcessor = WebRtc.videoProcessorFactory?.createVideoProcessor(videoSource) + val iosVideoTrack = WebRtc.peerConnectionFactory.videoTrackWithSource( + source = videoSource, + trackId = NSUUID.UUID().UUIDString() + ) + val videoCaptureController = CameraVideoCaptureController( + constraints = videoConstraints, + videoCapturerDelegate = videoProcessor ?: videoSource + ) LocalVideoStreamTrack(iosVideoTrack, videoCaptureController) } diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt new file mode 100644 index 0000000..90eb90e --- /dev/null +++ b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt @@ -0,0 +1,10 @@ +package com.shepeliev.webrtckmp + +import WebRTC.RTCVideoCapturerDelegateProtocol +import WebRTC.RTCVideoSource +import kotlinx.cinterop.ExperimentalForeignApi + +@OptIn(ExperimentalForeignApi::class) +fun interface VideoProcessorFactory { + fun createVideoProcessor(videoSource: RTCVideoSource): RTCVideoCapturerDelegateProtocol +} diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt index 3e3c31a..58e0735 100644 --- a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt +++ b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt @@ -16,6 +16,7 @@ object WebRtc { var videoDecoderFactory: RTCVideoDecoderFactoryProtocol? = null var peerConnectionFactoryOptions: RTCPeerConnectionFactoryOptions? = null var customPeerConnectionFactory: RTCPeerConnectionFactory? = null + var videoProcessorFactory: VideoProcessorFactory? = null internal val peerConnectionFactory: RTCPeerConnectionFactory by lazy { customPeerConnectionFactory ?: run {