Add video processor factory for Android and iOS targets (#125)

shepeliev · Aug 17, 2024 · 37f8067 · 37f8067
1 parent e2a33c8
commit 37f8067
Show file tree

Hide file tree

Showing 8 changed files with 37 additions and 9 deletions.
diff --git a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt
@@ -43,6 +43,7 @@ private object MediaDevicesImpl : MediaDevices {
         if (constraints.video != null) {
             checkCameraPermission()
             val videoSource = WebRtc.peerConnectionFactory.createVideoSource(false)
+            videoSource.setVideoProcessor(WebRtc.videoProcessorFactory?.createVideoProcessor())
             val videoCaptureController = CameraVideoCaptureController(
                 constraints.video,
                 videoSource
@@ -62,6 +63,7 @@ private object MediaDevicesImpl : MediaDevices {
 
     override suspend fun getDisplayMedia(): MediaStream {
         val videoSource = WebRtc.peerConnectionFactory.createVideoSource(false)
+        WebRtc.videoProcessorFactory?.createVideoProcessor()?.let { videoSource.setVideoProcessor(it) }
         val screenCaptureController = ScreenCaptureController(videoSource)
         val videoTrack = WebRtc.peerConnectionFactory.createVideoTrack(
             UUID.randomUUID().toString(),

diff --git a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt
@@ -0,0 +1,7 @@
+package com.shepeliev.webrtckmp
+
+import org.webrtc.VideoProcessor
+
+fun interface VideoProcessorFactory {
+    fun createVideoProcessor(): VideoProcessor
+}
diff --git a/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt b/webrtc-kmp/src/androidMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt
@@ -24,6 +24,8 @@ object WebRtc {
     var videoDecoderFactory: VideoDecoderFactory? = null
     var customCameraEnumerator: CameraEnumerator? = null
     var customPeerConnectionFactory: PeerConnectionFactory? = null
+    var videoProcessorFactory: VideoProcessorFactory? = null
+
     lateinit var factoryInitializationOptionsBuilder: PeerConnectionFactory.InitializationOptions.Builder
         private set
 

diff --git a/webrtc-kmp/src/commonMain/kotlin/com/shepeliev/webrtckmp/MediaTrackConstraints.kt b/webrtc-kmp/src/commonMain/kotlin/com/shepeliev/webrtckmp/MediaTrackConstraints.kt
@@ -162,8 +162,8 @@ class MediaTrackConstraintsBuilder(internal var constraints: MediaTrackConstrain
         constraints = constraints.copy(facingMode = constrain)
     }
 
-    fun frameRate(ratio: Double) {
-        constraints = constraints.copy(frameRate = ratio.asValueConstrain())
+    fun frameRate(fps: Double) {
+        constraints = constraints.copy(frameRate = fps.asValueConstrain())
     }
 
     fun frameRate(build: ValueOrConstrain.Constrain<Double>.() -> Unit) {

diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/CameraVideoCaptureController.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/CameraVideoCaptureController.kt
@@ -3,7 +3,7 @@
 package com.shepeliev.webrtckmp
 
 import WebRTC.RTCCameraVideoCapturer
-import WebRTC.RTCVideoSource
+import WebRTC.RTCVideoCapturerDelegateProtocol
 import kotlinx.cinterop.ExperimentalForeignApi
 import kotlinx.cinterop.useContents
 import platform.AVFoundation.AVCaptureDevice
@@ -19,7 +19,7 @@ import kotlin.math.abs
 
 internal class CameraVideoCaptureController(
     private val constraints: MediaTrackConstraints,
-    private val videoSource: RTCVideoSource,
+    private val videoCapturerDelegate: RTCVideoCapturerDelegateProtocol,
 ) : VideoCaptureController {
     private var videoCapturer: RTCCameraVideoCapturer? = null
     private var position: AVCaptureDevicePosition = AVCaptureDevicePositionBack
@@ -32,7 +32,7 @@ internal class CameraVideoCaptureController(
 
     override fun startCapture() {
         if (videoCapturer != null) return
-        videoCapturer = RTCCameraVideoCapturer(videoSource)
+        videoCapturer = RTCCameraVideoCapturer(videoCapturerDelegate)
         if (!this::device.isInitialized) selectDevice()
         selectFormat()
         selectFps()

diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/MediaDevices.kt
@@ -1,5 +1,3 @@
-@file:OptIn(ExperimentalForeignApi::class)
-
 package com.shepeliev.webrtckmp
 
 import WebRTC.RTCCameraVideoCapturer
@@ -10,6 +8,7 @@ import platform.Foundation.NSUUID
 
 internal actual val mediaDevices: MediaDevices = MediaDevicesImpl
 
+@OptIn(ExperimentalForeignApi::class)
 private object MediaDevicesImpl : MediaDevices {
     override suspend fun getUserMedia(streamConstraints: MediaStreamConstraintsBuilder.() -> Unit): MediaStream {
         val constraints = MediaStreamConstraintsBuilder().let {
@@ -29,8 +28,15 @@ private object MediaDevicesImpl : MediaDevices {
 
         val videoTrack = constraints.video?.let { videoConstraints ->
             val videoSource = WebRtc.peerConnectionFactory.videoSource()
-            val iosVideoTrack = WebRtc.peerConnectionFactory.videoTrackWithSource(videoSource, NSUUID.UUID().UUIDString())
-            val videoCaptureController = CameraVideoCaptureController(videoConstraints, videoSource)
+            val videoProcessor = WebRtc.videoProcessorFactory?.createVideoProcessor(videoSource)
+            val iosVideoTrack = WebRtc.peerConnectionFactory.videoTrackWithSource(
+                source = videoSource,
+                trackId = NSUUID.UUID().UUIDString()
+            )
+            val videoCaptureController = CameraVideoCaptureController(
+                constraints = videoConstraints,
+                videoCapturerDelegate = videoProcessor ?: videoSource
+            )
             LocalVideoStreamTrack(iosVideoTrack, videoCaptureController)
         }
 

diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/VideoProcessorFactory.kt
@@ -0,0 +1,10 @@
+package com.shepeliev.webrtckmp
+
+import WebRTC.RTCVideoCapturerDelegateProtocol
+import WebRTC.RTCVideoSource
+import kotlinx.cinterop.ExperimentalForeignApi
+
+@OptIn(ExperimentalForeignApi::class)
+fun interface VideoProcessorFactory {
+    fun createVideoProcessor(videoSource: RTCVideoSource): RTCVideoCapturerDelegateProtocol
+}
diff --git a/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt b/webrtc-kmp/src/iosMain/kotlin/com/shepeliev/webrtckmp/WebRtc.kt
@@ -16,6 +16,7 @@ object WebRtc {
     var videoDecoderFactory: RTCVideoDecoderFactoryProtocol? = null
     var peerConnectionFactoryOptions: RTCPeerConnectionFactoryOptions? = null
     var customPeerConnectionFactory: RTCPeerConnectionFactory? = null
+    var videoProcessorFactory: VideoProcessorFactory? = null
 
     internal val peerConnectionFactory: RTCPeerConnectionFactory by lazy {
         customPeerConnectionFactory ?: run {