Allow custom audio processing by exposing AudioProcessingModule

sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h

@@ -73,12 +73,6 @@ NS_ASSUME_NONNULL_BEGIN
     initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
             decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
 
-- (instancetype)
-    initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
-                   encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
-
 /** Initialize an RTCPeerConnection with a configuration, constraints, and
  *  dependencies.
  */

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h

@@ -38,6 +38,8 @@ typedef NS_ENUM(NSInteger, RTCRtpMediaType);
 (RTCSSLCertificateVerifier);
 @protocol RTC_OBJC_TYPE
 (RTCAudioDevice);
+@protocol RTC_OBJC_TYPE
+(RTCAudioProcessingModule);
 
 RTC_OBJC_EXPORT
 @interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) : NSObject
@@ -60,8 +62,9 @@ RTC_OBJC_EXPORT
 - (instancetype)
     initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
                    encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
+                   decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioProcessingModule:
+                (nullable id<RTC_OBJC_TYPE(RTCAudioProcessingModule)>)audioProcessingModule;
 
 @property(nonatomic, readonly) RTCAudioDeviceModule *audioDeviceModule;
 

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm

@@ -53,6 +53,9 @@
 #include "sdk/objc/native/src/objc_video_decoder_factory.h"
 #include "sdk/objc/native/src/objc_video_encoder_factory.h"
 
+#import "components/audio/RTCAudioProcessingModule.h"
+#import "components/audio/RTCDefaultAudioProcessingModule+Private.h"
+
 #if defined(WEBRTC_IOS)
 #import "sdk/objc/native/api/audio_device_module.h"
 #endif
@@ -62,6 +65,7 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) {
   std::unique_ptr<rtc::Thread> _workerThread;
   std::unique_ptr<rtc::Thread> _signalingThread;
   rtc::scoped_refptr<webrtc::AudioDeviceModule> _nativeAudioDeviceModule;
+  RTCDefaultAudioProcessingModule *_defaultAudioProcessingModule;
 
   BOOL _hasStartedAecDump;
 }
@@ -144,8 +148,9 @@ - (instancetype)init {
 - (instancetype)
     initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
                    encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory {
+                   decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioProcessingModule:
+                (nullable id<RTC_OBJC_TYPE(RTCAudioProcessingModule)>)audioProcessingModule {
 #ifdef HAVE_NO_MEDIA
   return [self initWithNoMedia];
 #else
@@ -158,12 +163,21 @@ - (instancetype)init {
     native_decoder_factory = webrtc::ObjCToNativeVideoDecoderFactory(decoderFactory);
   }
   rtc::scoped_refptr<webrtc::AudioDeviceModule> audio_device_module = [self createAudioDeviceModule:bypassVoiceProcessing];
+
+  if ([audioProcessingModule isKindOfClass:[RTCDefaultAudioProcessingModule class]]) {
+    _defaultAudioProcessingModule = (RTCDefaultAudioProcessingModule *)audioProcessingModule;
+  } else {
+    _defaultAudioProcessingModule = [[RTCDefaultAudioProcessingModule alloc] init];
+  }
+
+  NSLog(@"AudioProcessingModule: %@", _defaultAudioProcessingModule);
+
   return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory()
                        nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory()
                        nativeVideoEncoderFactory:std::move(native_encoder_factory)
                        nativeVideoDecoderFactory:std::move(native_decoder_factory)
                                audioDeviceModule:audio_device_module.get()
-                           audioProcessingModule:nullptr
+                           audioProcessingModule:_defaultAudioProcessingModule.nativeAudioProcessingModule
                            bypassVoiceProcessing:bypassVoiceProcessing];
 #endif
 }
@@ -268,11 +282,11 @@ - (instancetype)initWithNativeAudioEncoderFactory:
     media_deps.audio_decoder_factory = std::move(audioDecoderFactory);
     media_deps.video_encoder_factory = std::move(videoEncoderFactory);
     media_deps.video_decoder_factory = std::move(videoDecoderFactory);
-    if (audioProcessingModule) {
-      media_deps.audio_processing = std::move(audioProcessingModule);
-    } else {
-      media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create();
-    }
+    // if (audioProcessingModule) {
+    media_deps.audio_processing = std::move(audioProcessingModule);
+    // } else {
+    //   media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create();
+    // }
     media_deps.trials = dependencies.trials.get();
     dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps));
     dependencies.call_factory = webrtc::CreateCallFactory();

sdk/objc/components/audio/RTCAudioBuffer+Private.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioBuffer.h"
+
+#include "modules/audio_processing/audio_buffer.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTC_OBJC_TYPE (RTCAudioBuffer)()
+
+- (instancetype)initWithNativeType: (webrtc::AudioBuffer *) audioBuffer;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioBuffer.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCAudioBuffer) : NSObject
+
+@property(nonatomic, readonly) size_t channels;
+@property(nonatomic, readonly) size_t frames;
+@property(nonatomic, readonly) size_t framesPerBand;
+@property(nonatomic, readonly) size_t bands;
+
+// Returns pointer arrays. Index range from 0 to `frames`.
+- (float* _Nonnull)rawBufferForChannel:(size_t)channel;
+
+// TODO: More convenience methods...
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioBuffer.mm

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioBuffer.h"
+
+#include "modules/audio_processing/audio_buffer.h"
+
+@implementation RTC_OBJC_TYPE (RTCAudioBuffer) {
+  // Raw
+  webrtc::AudioBuffer *_audioBuffer;
+}
+
+- (size_t)channels {
+  return _audioBuffer->num_channels();
+}
+
+- (size_t)frames {
+  return _audioBuffer->num_frames();
+}
+
+- (size_t)framesPerBand {
+  return _audioBuffer->num_frames_per_band();
+}
+
+- (size_t)bands {
+  return _audioBuffer->num_bands();
+}
+
+- (float *)rawBufferForChannel:(size_t)channel {
+  return _audioBuffer->channels()[channel];
+}
+
+#pragma mark - Private
+
+- (instancetype)initWithNativeType:(webrtc::AudioBuffer *)audioBuffer {
+  if (self = [super init]) {
+    _audioBuffer = audioBuffer;
+  }
+  return self;
+}
+
+@end

sdk/objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioCustomProcessingAdapter.h"
+#import "RTCAudioCustomProcessingDelegate.h"
+#import "RTCMacros.h"
+
+#include "modules/audio_processing/include/audio_processing.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTCAudioCustomProcessingAdapter ()
+
+// Thread safe set/get with os_unfair_lock.
+@property(nonatomic, nullable) id<RTCAudioCustomProcessingDelegate> audioCustomProcessingDelegate;
+
+// Direct read access without lock.
+@property(nonatomic, readonly, nullable) id<RTCAudioCustomProcessingDelegate>
+    rawAudioCustomProcessingDelegate;
+
+@property(nonatomic, readonly) std::unique_ptr<webrtc::CustomProcessing>
+    nativeAudioCustomProcessingModule;
+
+- (instancetype)initWithDelegate:
+    (nullable id<RTCAudioCustomProcessingDelegate>)audioCustomProcessingDelegate;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTCAudioCustomProcessingAdapter : NSObject
+
+- (instancetype)init NS_UNAVAILABLE;
+
+@end
+
+NS_ASSUME_NONNULL_END