From 8bd399143514dbbf2a782f9fcab8b9a26a8a864d Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 24 Nov 2022 02:58:55 +0900 Subject: [PATCH] AudioUnit: Don't rely on category switch for mic indicator to turn off (#52) * progress * tweak * clean * simplify audio unit restart call to SetupAudioBuffersForActiveAudioSession() might not be needed since sample rate won't change during restart. This might help reduce the unwanted noise when restarting audio unit. * clean --- .../audio/RTCAudioSession+Private.h | 2 - sdk/objc/components/audio/RTCAudioSession.h | 3 - sdk/objc/components/audio/RTCAudioSession.mm | 18 --- .../RTCNativeAudioSessionDelegateAdapter.mm | 5 - sdk/objc/native/src/audio/audio_device_ios.h | 16 ++- sdk/objc/native/src/audio/audio_device_ios.mm | 120 +++++++++--------- .../native/src/audio/audio_session_observer.h | 2 - .../src/audio/voice_processing_audio_unit.h | 2 +- .../src/audio/voice_processing_audio_unit.mm | 18 +-- 9 files changed, 76 insertions(+), 110 deletions(-) diff --git a/sdk/objc/components/audio/RTCAudioSession+Private.h b/sdk/objc/components/audio/RTCAudioSession+Private.h index f7bc1acb89..2be1b9fb3d 100644 --- a/sdk/objc/components/audio/RTCAudioSession+Private.h +++ b/sdk/objc/components/audio/RTCAudioSession+Private.h @@ -35,8 +35,6 @@ NS_ASSUME_NONNULL_BEGIN */ @property(nonatomic, assign) BOOL isInterrupted; -@property(nonatomic, strong) NSString *activeCategory; - /** Adds the delegate to the list of delegates, and places it at the front of * the list. This delegate will be notified before other delegates of * audio events. diff --git a/sdk/objc/components/audio/RTCAudioSession.h b/sdk/objc/components/audio/RTCAudioSession.h index 99cd004b4d..3b83b27ba5 100644 --- a/sdk/objc/components/audio/RTCAudioSession.h +++ b/sdk/objc/components/audio/RTCAudioSession.h @@ -102,9 +102,6 @@ RTC_OBJC_EXPORT - (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession audioUnitStartFailedWithError:(NSError *)error; -/** Called when audio session changed from output-only to input & output */ -- (void)audioSessionDidChangeRecordingEnabled:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession; - @end /** This is a protocol used to inform RTCAudioSession when the audio session diff --git a/sdk/objc/components/audio/RTCAudioSession.mm b/sdk/objc/components/audio/RTCAudioSession.mm index 11772e96d0..02e12113c4 100644 --- a/sdk/objc/components/audio/RTCAudioSession.mm +++ b/sdk/objc/components/audio/RTCAudioSession.mm @@ -114,8 +114,6 @@ - (instancetype)initWithAudioSession:(id)audioSession { options:NSKeyValueObservingOptionNew | NSKeyValueObservingOptionOld context:(__bridge void *)RTC_OBJC_TYPE(RTCAudioSession).class]; - _activeCategory = _session.category; - RTCLog(@"RTC_OBJC_TYPE(RTCAudioSession) (%p): init.", self); } return self; @@ -543,13 +541,6 @@ - (void)handleRouteChangeNotification:(NSNotification *)notification { break; case AVAudioSessionRouteChangeReasonCategoryChange: RTCLog(@"Audio route changed: CategoryChange to :%@", self.session.category); - { - if (![_session.category isEqualToString:_activeCategory]) { - _activeCategory = _session.category; - RTCLog(@"Audio route changed: Restarting Audio Unit"); - [self notifyDidChangeAudioSessionRecordingEnabled]; - } - } break; case AVAudioSessionRouteChangeReasonOverride: RTCLog(@"Audio route changed: Override"); @@ -1005,13 +996,4 @@ - (void)notifyFailedToSetActive:(BOOL)active error:(NSError *)error { } } -- (void)notifyDidChangeAudioSessionRecordingEnabled { - for (auto delegate : self.delegates) { - SEL sel = @selector(audioSessionDidChangeRecordingEnabled:); - if ([delegate respondsToSelector:sel]) { - [delegate audioSessionDidChangeRecordingEnabled:self]; - } - } -} - @end diff --git a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm index 6f8978b066..daddf314a4 100644 --- a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm +++ b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm @@ -86,9 +86,4 @@ - (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession _observer->OnChangedOutputVolume(); } -- (void)audioSessionDidChangeRecordingEnabled:(RTC_OBJC_TYPE(RTCAudioSession) *)session { - // re-trigger audio unit init, by using interrupt ended callback - _observer->OnChangedRecordingEnabled(); -} - @end diff --git a/sdk/objc/native/src/audio/audio_device_ios.h b/sdk/objc/native/src/audio/audio_device_ios.h index f2792a9f48..f8d9440a49 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.h +++ b/sdk/objc/native/src/audio/audio_device_ios.h @@ -147,7 +147,6 @@ class AudioDeviceIOS : public AudioDeviceGeneric, void OnValidRouteChange() override; void OnCanPlayOrRecordChange(bool can_play_or_record) override; void OnChangedOutputVolume() override; - void OnChangedRecordingEnabled() override; // VoiceProcessingAudioUnitObserver methods. OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, @@ -175,7 +174,8 @@ class AudioDeviceIOS : public AudioDeviceGeneric, void HandleSampleRateChange(); void HandlePlayoutGlitchDetected(); void HandleOutputVolumeChange(); - void HandleAudioSessionRecordingEnabledChange(); + + bool RestartAudioUnit(bool enable_input); // Uses current `playout_parameters_` and `record_parameters_` to inform the // audio device buffer (ADB) about our internal audio parameters. @@ -205,7 +205,7 @@ class AudioDeviceIOS : public AudioDeviceGeneric, // Activates our audio session, creates and initializes the voice-processing // audio unit and verifies that we got the preferred native audio parameters. - bool InitPlayOrRecord(); + bool InitPlayOrRecord(bool enable_input); // Closes and deletes the voice-processing I/O unit. void ShutdownPlayOrRecord(); @@ -269,19 +269,21 @@ class AudioDeviceIOS : public AudioDeviceGeneric, // will be changed dynamically to account for this behavior. rtc::BufferT record_audio_buffer_; + // Set to 1 when recording is initialized and 0 otherwise. + volatile int recording_is_initialized_; + // Set to 1 when recording is active and 0 otherwise. volatile int recording_; + // Set to 1 when playout is initialized and 0 otherwise. + volatile int playout_is_initialized_; + // Set to 1 when playout is active and 0 otherwise. volatile int playing_; // Set to true after successful call to Init(), false otherwise. bool initialized_ RTC_GUARDED_BY(thread_checker_); - // Set to true after successful call to InitRecording() or InitPlayout(), - // false otherwise. - bool audio_is_initialized_; - // Set to true if audio session is interrupted, false otherwise. bool is_interrupted_; diff --git a/sdk/objc/native/src/audio/audio_device_ios.mm b/sdk/objc/native/src/audio/audio_device_ios.mm index dc804d3c77..a106b78e63 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_ios.mm @@ -68,7 +68,6 @@ kMessageTypeCanPlayOrRecordChange, kMessageTypePlayoutGlitchDetected, kMessageOutputVolumeChange, - kMessageTypeRecordingEnabledChange, }; using ios::CheckAndLogError; @@ -104,10 +103,11 @@ static void LogDeviceInfo() { : bypass_voice_processing_(bypass_voice_processing), audio_device_buffer_(nullptr), audio_unit_(nullptr), + recording_is_initialized_(0), recording_(0), + playout_is_initialized_(0), playing_(0), initialized_(false), - audio_is_initialized_(false), is_interrupted_(false), has_configured_session_(false), num_detected_playout_glitches_(0), @@ -188,48 +188,59 @@ static void LogDeviceInfo() { LOGI() << "InitPlayout"; RTC_DCHECK_RUN_ON(&thread_checker_); RTC_DCHECK(initialized_); - RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!playout_is_initialized_); RTC_DCHECK(!playing_); - if (!audio_is_initialized_) { - if (!InitPlayOrRecord()) { + + if (!recording_is_initialized_) { + // recording not initialized yet, init with no input + if (!InitPlayOrRecord(false)) { RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitPlayout!"; return -1; } } - audio_is_initialized_ = true; + + rtc::AtomicOps::ReleaseStore(&playout_is_initialized_, 1); + return 0; } bool AudioDeviceIOS::PlayoutIsInitialized() const { RTC_DCHECK_RUN_ON(&thread_checker_); - return audio_is_initialized_; + return playout_is_initialized_; } bool AudioDeviceIOS::RecordingIsInitialized() const { RTC_DCHECK_RUN_ON(&thread_checker_); - return audio_is_initialized_; + return recording_is_initialized_; } int32_t AudioDeviceIOS::InitRecording() { LOGI() << "InitRecording"; RTC_DCHECK_RUN_ON(&thread_checker_); RTC_DCHECK(initialized_); - RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!recording_is_initialized_); RTC_DCHECK(!recording_); - if (!audio_is_initialized_) { - if (!InitPlayOrRecord()) { + + if (!playout_is_initialized_) { + // playout not initialized yet, init with input + if (!InitPlayOrRecord(true)) { RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitRecording!"; return -1; } + } else { + // playout already initialized, restart audio unit with input + RestartAudioUnit(true); } - audio_is_initialized_ = true; + + rtc::AtomicOps::ReleaseStore(&recording_is_initialized_, 1); + return 0; } int32_t AudioDeviceIOS::StartPlayout() { LOGI() << "StartPlayout"; RTC_DCHECK_RUN_ON(&thread_checker_); - RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(playout_is_initialized_); RTC_DCHECK(!playing_); RTC_DCHECK(audio_unit_); if (fine_audio_buffer_) { @@ -254,14 +265,19 @@ static void LogDeviceInfo() { int32_t AudioDeviceIOS::StopPlayout() { LOGI() << "StopPlayout"; RTC_DCHECK_RUN_ON(&thread_checker_); - if (!audio_is_initialized_ || !playing_) { + + if (!playout_is_initialized_ || !playing_) { return 0; } + if (!recording_) { ShutdownPlayOrRecord(); - audio_is_initialized_ = false; + + rtc::AtomicOps::ReleaseStore(&recording_is_initialized_, 0); } + rtc::AtomicOps::ReleaseStore(&playing_, 0); + rtc::AtomicOps::ReleaseStore(&playout_is_initialized_, 0); // Derive average number of calls to OnGetPlayoutData() between detected // audio glitches and add the result to a histogram. @@ -285,7 +301,7 @@ static void LogDeviceInfo() { int32_t AudioDeviceIOS::StartRecording() { LOGI() << "StartRecording"; RTC_DCHECK_RUN_ON(&thread_checker_); - RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(recording_is_initialized_); RTC_DCHECK(!recording_); RTC_DCHECK(audio_unit_); if (fine_audio_buffer_) { @@ -308,14 +324,23 @@ static void LogDeviceInfo() { int32_t AudioDeviceIOS::StopRecording() { LOGI() << "StopRecording"; RTC_DCHECK_RUN_ON(&thread_checker_); - if (!audio_is_initialized_ || !recording_) { + + if (!recording_is_initialized_ || !recording_) { return 0; } + if (!playing_) { ShutdownPlayOrRecord(); - audio_is_initialized_ = false; + + rtc::AtomicOps::ReleaseStore(&playout_is_initialized_, 0); + } else if (playout_is_initialized_) { + // restart audio unit with no input + RestartAudioUnit(false); } + rtc::AtomicOps::ReleaseStore(&recording_, 0); + rtc::AtomicOps::ReleaseStore(&recording_is_initialized_, 0); + return 0; } @@ -374,11 +399,6 @@ static void LogDeviceInfo() { thread_->Post(RTC_FROM_HERE, this, kMessageOutputVolumeChange); } -void AudioDeviceIOS::OnChangedRecordingEnabled() { - RTC_DCHECK(thread_); - thread_->Post(RTC_FROM_HERE, this, kMessageTypeRecordingEnabledChange); -} - OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, const AudioTimeStamp* time_stamp, UInt32 bus_number, @@ -509,9 +529,6 @@ static void LogDeviceInfo() { case kMessageOutputVolumeChange: HandleOutputVolumeChange(); break; - case kMessageTypeRecordingEnabledChange: - HandleAudioSessionRecordingEnabledChange(); - break; } } @@ -627,7 +644,7 @@ static void LogDeviceInfo() { SetupAudioBuffersForActiveAudioSession(); // Initialize the audio unit again with the new sample rate. - if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + if (!audio_unit_->Initialize(playout_parameters_.sample_rate(), recording_is_initialized_)) { RTCLogError(@"Failed to initialize the audio unit with sample rate: %d", playout_parameters_.sample_rate()); return; @@ -681,59 +698,44 @@ static void LogDeviceInfo() { last_output_volume_change_time_ = rtc::TimeMillis(); } -void AudioDeviceIOS::HandleAudioSessionRecordingEnabledChange() { +bool AudioDeviceIOS::RestartAudioUnit(bool enable_input) { RTC_DCHECK_RUN_ON(&thread_checker_); - LOGI() << "HandleAudioSessionRecordingEnabledChange"; + LOGI() << "RestartAudioUnit"; // If we don't have an audio unit yet, or the audio unit is uninitialized, // there is no work to do. if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { - return; + return false; } - // The audio unit is already initialized or started. - // Check to see if the sample rate or buffer size has changed. - RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; - const double session_sample_rate = session.sampleRate; - - // Extra sanity check to ensure that the new sample rate is valid. - if (session_sample_rate <= 0.0) { - RTCLogError(@"Sample rate is invalid: %f", session_sample_rate); - LOGI() << "Sample rate is invalid " << session_sample_rate; - return; - } - // We need to adjust our format and buffer sizes. - // The stream format is about to be changed and it requires that we first - // stop and uninitialize the audio unit to deallocate its resources. - RTCLog(@"Stopping and uninitializing audio unit to adjust buffers."); bool restart_audio_unit = false; if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { audio_unit_->Stop(); - restart_audio_unit = true; PrepareForNewStart(); + restart_audio_unit = true; } + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { audio_unit_->Uninitialize(); } - // Allocate new buffers given the new stream format. - SetupAudioBuffersForActiveAudioSession(); + // Initialize the audio unit again with the same sample rate. + const double sample_rate = playout_parameters_.sample_rate(); - // Initialize the audio unit again with the new sample rate. - RTC_DCHECK_EQ(playout_parameters_.sample_rate(), session_sample_rate); - if (!audio_unit_->Initialize(session_sample_rate)) { - RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", session_sample_rate); - return; + if (!audio_unit_->Initialize(sample_rate, enable_input)) { + RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", sample_rate); + return false; } // Restart the audio unit if it was already running. if (restart_audio_unit && !audio_unit_->Start()) { - RTCLogError(@"Failed to start audio unit with sample rate: %f", session_sample_rate); - return; + RTCLogError(@"Failed to start audio unit with sample rate: %f", sample_rate); + return false; } LOGI() << "Successfully enabled audio unit for recording."; + return true; } void AudioDeviceIOS::UpdateAudioDeviceBuffer() { @@ -829,7 +831,7 @@ static void LogDeviceInfo() { // If we're not initialized we don't need to do anything. Audio unit will // be initialized on initialization. - if (!audio_is_initialized_) return; + if (!playout_is_initialized_ && !recording_is_initialized_) return; // If we're initialized, we must have an audio unit. RTC_DCHECK(audio_unit_); @@ -866,7 +868,7 @@ static void LogDeviceInfo() { RTCLog(@"Initializing audio unit for UpdateAudioUnit"); ConfigureAudioSession(); SetupAudioBuffersForActiveAudioSession(); - if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + if (!audio_unit_->Initialize(playout_parameters_.sample_rate(), recording_is_initialized_)) { RTCLogError(@"Failed to initialize audio unit."); return; } @@ -956,7 +958,7 @@ static void LogDeviceInfo() { RTCLog(@"Unconfigured audio session."); } -bool AudioDeviceIOS::InitPlayOrRecord() { +bool AudioDeviceIOS::InitPlayOrRecord(bool enable_input) { LOGI() << "InitPlayOrRecord"; RTC_DCHECK_RUN_ON(&thread_checker_); @@ -992,7 +994,7 @@ static void LogDeviceInfo() { return false; } SetupAudioBuffersForActiveAudioSession(); - audio_unit_->Initialize(playout_parameters_.sample_rate()); + audio_unit_->Initialize(playout_parameters_.sample_rate(), enable_input); } // Release the lock. diff --git a/sdk/objc/native/src/audio/audio_session_observer.h b/sdk/objc/native/src/audio/audio_session_observer.h index 978d8e2cdd..f7c44c8184 100644 --- a/sdk/objc/native/src/audio/audio_session_observer.h +++ b/sdk/objc/native/src/audio/audio_session_observer.h @@ -32,8 +32,6 @@ class AudioSessionObserver { virtual void OnChangedOutputVolume() = 0; - virtual void OnChangedRecordingEnabled() = 0; - protected: virtual ~AudioSessionObserver() {} }; diff --git a/sdk/objc/native/src/audio/voice_processing_audio_unit.h b/sdk/objc/native/src/audio/voice_processing_audio_unit.h index ed9dd98568..b474cda104 100644 --- a/sdk/objc/native/src/audio/voice_processing_audio_unit.h +++ b/sdk/objc/native/src/audio/voice_processing_audio_unit.h @@ -75,7 +75,7 @@ class VoiceProcessingAudioUnit { VoiceProcessingAudioUnit::State GetState() const; // Initializes the underlying audio unit with the given sample rate. - bool Initialize(Float64 sample_rate); + bool Initialize(Float64 sample_rate, bool enable_input); // Starts the underlying audio unit. OSStatus Start(); diff --git a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm index 1663b7145c..b3daacb334 100644 --- a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm +++ b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm @@ -180,7 +180,7 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return state_; } -bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) { +bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate, bool enable_input) { RTC_DCHECK_GE(state_, kUninitialized); RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate); @@ -191,19 +191,11 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { LogStreamDescription(format); #endif - // Enable input on the input scope of the input element. - // keep it disabled if audio session is configured for playback only - AVAudioSession* session = [AVAudioSession sharedInstance]; - UInt32 enable_input = 0; - if ([session.category isEqualToString: AVAudioSessionCategoryPlayAndRecord] || - [session.category isEqualToString: AVAudioSessionCategoryRecord]) { - enable_input = 1; - } - RTCLog(@"Initializing AudioUnit, category=%@, enable_input=%d", session.category, (int) enable_input); - // LOGI() << "Initialize" << session.category << ", enable_input=" << enable_input; + UInt32 _enable_input = enable_input ? 1 : 0; + RTCLog(@"Initializing AudioUnit, _enable_input=%d", (int) _enable_input); result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, - kAudioUnitScope_Input, kInputBus, &enable_input, - sizeof(enable_input)); + kAudioUnitScope_Input, kInputBus, &_enable_input, + sizeof(_enable_input)); if (result != noErr) { DisposeAudioUnit(); RTCLogError(@"Failed to enable input on input scope of input element. "