diff --git a/Android/app/build.gradle b/Android/app/build.gradle index 9068acc..3f5f0ff 100644 --- a/Android/app/build.gradle +++ b/Android/app/build.gradle @@ -4,15 +4,15 @@ plugins { } android { - compileSdkVersion 30 + compileSdkVersion 31 buildToolsVersion "30.0.3" defaultConfig { applicationId "com.example.microphone" minSdkVersion 23 - targetSdkVersion 30 + targetSdkVersion 31 versionCode 6 - versionName "1.6" + versionName "1.7" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } @@ -35,13 +35,13 @@ android { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" - implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.3' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.1' implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.3.3' - implementation 'androidx.core:core-ktx:1.3.1' - implementation 'androidx.appcompat:appcompat:1.2.0' - implementation 'com.google.android.material:material:1.2.1' - implementation 'androidx.constraintlayout:constraintlayout:2.0.1' - testImplementation 'junit:junit:4.+' - androidTestImplementation 'androidx.test.ext:junit:1.1.2' - androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' + implementation 'androidx.core:core-ktx:1.7.0' + implementation 'androidx.appcompat:appcompat:1.4.1' + implementation 'com.google.android.material:material:1.6.0' + implementation 'androidx.constraintlayout:constraintlayout:2.1.3' + testImplementation 'junit:junit:' + androidTestImplementation 'androidx.test.ext:junit:1.1.3' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0' } \ No newline at end of file diff --git a/Android/app/src/main/AndroidManifest.xml b/Android/app/src/main/AndroidManifest.xml index 687a1ce..ba805a5 100644 --- a/Android/app/src/main/AndroidManifest.xml +++ b/Android/app/src/main/AndroidManifest.xml @@ -3,6 +3,7 @@ package="com.example.microphone"> + @@ -16,7 +17,8 @@ android:theme="@style/Theme.Microphone" android:name=".DefaultApp"> + android:name=".MainActivity" + android:exported="true"> diff --git a/Android/app/src/main/java/com/example/microphone/MainActivity.kt b/Android/app/src/main/java/com/example/microphone/MainActivity.kt index 5a2c383..9434250 100644 --- a/Android/app/src/main/java/com/example/microphone/MainActivity.kt +++ b/Android/app/src/main/java/com/example/microphone/MainActivity.kt @@ -91,6 +91,12 @@ class MainActivity : AppCompatActivity() // check for audio permission if(ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.RECORD_AUDIO), 0) + if(ContextCompat.checkSelfPermission(this, Manifest.permission.BLUETOOTH) != PackageManager.PERMISSION_GRANTED) + ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.BLUETOOTH), 0) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + if(ContextCompat.checkSelfPermission(this, Manifest.permission.BLUETOOTH_CONNECT) != PackageManager.PERMISSION_GRANTED) + ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.BLUETOOTH_CONNECT), 0) + } // start handler thread handlerThread = HandlerThread("MicActivityStart", Process.THREAD_PRIORITY_BACKGROUND) handlerThread.start() diff --git a/Android/app/src/main/java/com/example/microphone/audio/AudioBuffer.kt b/Android/app/src/main/java/com/example/microphone/audio/AudioBuffer.kt index 5e3872e..6959fee 100644 --- a/Android/app/src/main/java/com/example/microphone/audio/AudioBuffer.kt +++ b/Android/app/src/main/java/com/example/microphone/audio/AudioBuffer.kt @@ -6,7 +6,7 @@ import kotlinx.coroutines.sync.withLock class AudioBuffer { // set buffer size for latency - private val BUFFER_SIZE = 2 + private val BUFFER_SIZE = 3 // actual buffer of byte arrays (FIFO with queue) private val buffer = ArrayDeque() // mutex for coroutines diff --git a/Android/app/src/main/java/com/example/microphone/audio/MicAudioManager.kt b/Android/app/src/main/java/com/example/microphone/audio/MicAudioManager.kt index f7fe6ca..ae10c13 100644 --- a/Android/app/src/main/java/com/example/microphone/audio/MicAudioManager.kt +++ b/Android/app/src/main/java/com/example/microphone/audio/MicAudioManager.kt @@ -4,7 +4,6 @@ import android.Manifest import android.content.Context import android.content.pm.PackageManager import android.media.AudioFormat -import android.media.AudioManager import android.media.AudioRecord import android.media.MediaRecorder import android.util.Log @@ -42,8 +41,6 @@ class MicAudioManager(ctx : Context) { require(ContextCompat.checkSelfPermission(ctx, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED){ "Microphone recording is not permitted" } - // setup noise suppression - (ctx.getSystemService(Context.AUDIO_SERVICE) as AudioManager).setParameters("noise_suppression=auto") // init recorder recorder = AudioRecord(AUDIO_SOURCE, SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT, BUFFER_SIZE) require(recorder?.state == AudioRecord.STATE_INITIALIZED){"Microphone not init properly"} @@ -53,7 +50,7 @@ class MicAudioManager(ctx : Context) { suspend fun record(audioBuffer : AudioBuffer) { // read number of shorts - val size = recorder?.read(buffer, 0, BUFFER_SIZE, AudioRecord.READ_BLOCKING) ?: return + val size = recorder?.read(buffer, 0, BUFFER_SIZE) ?: return if(size <= 0) { delay(RECORD_DELAY) diff --git a/Android/app/src/main/java/com/example/microphone/service/ForegroundService.kt b/Android/app/src/main/java/com/example/microphone/service/ForegroundService.kt index fa24e8b..930c216 100644 --- a/Android/app/src/main/java/com/example/microphone/service/ForegroundService.kt +++ b/Android/app/src/main/java/com/example/microphone/service/ForegroundService.kt @@ -379,7 +379,7 @@ class ForegroundService : Service() { val onTap = Intent(ctx, MainActivity::class.java).apply { addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_BROUGHT_TO_FRONT) } - val pendingIntent = PendingIntent.getActivity(ctx, 0, onTap, 0) + val pendingIntent = PendingIntent.getActivity(ctx, 0, onTap, PendingIntent.FLAG_IMMUTABLE) val builder = NotificationCompat.Builder(ctx, "AndroidMic") .setSmallIcon(R.drawable.icon) .setContentTitle(getString(R.string.activity_name)) diff --git a/Android/app/src/main/java/com/example/microphone/streaming/BluetoothStreamer.kt b/Android/app/src/main/java/com/example/microphone/streaming/BluetoothStreamer.kt index abd240a..5f11d09 100644 --- a/Android/app/src/main/java/com/example/microphone/streaming/BluetoothStreamer.kt +++ b/Android/app/src/main/java/com/example/microphone/streaming/BluetoothStreamer.kt @@ -1,15 +1,13 @@ package com.example.microphone.streaming import android.Manifest -import android.bluetooth.BluetoothAdapter -import android.bluetooth.BluetoothClass -import android.bluetooth.BluetoothDevice -import android.bluetooth.BluetoothSocket +import android.bluetooth.* import android.content.BroadcastReceiver import android.content.Context import android.content.Intent import android.content.IntentFilter import android.content.pm.PackageManager +import android.os.Build import android.util.Log import androidx.core.content.ContextCompat import com.example.microphone.audio.AudioBuffer @@ -26,7 +24,7 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { private val myUUID : UUID = UUID.fromString("34335e34-bccf-11eb-8529-0242ac130003") private val MAX_WAIT_TIME = 1500L // timeout - private val adapter : BluetoothAdapter? = BluetoothAdapter.getDefaultAdapter() + private val adapter : BluetoothAdapter private var target : BluetoothDevice? = null private var socket : BluetoothSocket? = null @@ -51,12 +49,20 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { // init everything init { + // get bluetooth adapter + val bm = ctx.getSystemService(Context.BLUETOOTH_SERVICE) as BluetoothManager + adapter = bm.adapter // check bluetooth adapter require(adapter != null) {"Bluetooth adapter is not found"} // check permission require(ContextCompat.checkSelfPermission(ctx, Manifest.permission.BLUETOOTH) == PackageManager.PERMISSION_GRANTED){ "Bluetooth is not permitted" } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + require(ContextCompat.checkSelfPermission(ctx, Manifest.permission.BLUETOOTH_CONNECT) == PackageManager.PERMISSION_GRANTED){ + "Bluetooth is not permitted" + } + } require(adapter.isEnabled){"Bluetooth adapter is not enabled"} // set target device selectTargetDevice() @@ -77,6 +83,10 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { } catch (e : IOException) { Log.d(TAG, "connect [createInsecureRfcommSocketToServiceRecord]: ${e.message}") null + } catch (e : SecurityException) + { + Log.d(TAG, "connect [createInsecureRfcommSocketToServiceRecord]: ${e.message}") + null } ?: return false // connect to server try { @@ -84,6 +94,10 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { } catch (e : IOException){ Log.d(TAG, "connect [connect]: ${e.message}") return false + } catch (e : SecurityException) + { + Log.d(TAG, "connect [connect]: ${e.message}") + return false } Log.d(TAG, "connect: connected") return true @@ -136,22 +150,28 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { private fun selectTargetDevice() { target = null - val pairedDevices = adapter?.bondedDevices ?: return - for(device in pairedDevices) - { - if(device.bluetoothClass.majorDeviceClass == BluetoothClass.Device.Major.COMPUTER) + try { + val pairedDevices = adapter.bondedDevices ?: return + for(device in pairedDevices) { - Log.d(TAG, "selectTargetDevice: testing ${device.name}") - if(testConnection(device)) + if(device.bluetoothClass.majorDeviceClass == BluetoothClass.Device.Major.COMPUTER) { - target = device - Log.d(TAG, "selectTargetDevice: ${device.name} is valid") - break + Log.d(TAG, "selectTargetDevice: testing ${device.name}") + if(testConnection(device)) + { + target = device + Log.d(TAG, "selectTargetDevice: ${device.name} is valid") + break + } + else + Log.d(TAG, "selectTargetDevice: ${device.name} is invalid") } - else - Log.d(TAG, "selectTargetDevice: ${device.name} is invalid") } + } catch(e : SecurityException) + { + Log.d(TAG, "selectTargetDevice: ${e.message}") } + } // test connection with a device @@ -165,6 +185,10 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { } catch (e : IOException) { Log.d(TAG, "testConnection [createInsecureRfcommSocketToServiceRecord]: ${e.message}") null + } catch (e : SecurityException) + { + Log.d(TAG, "testConnection [createInsecureRfcommSocketToServiceRecord]: ${e.message}") + null } ?: return false // try to connect try { @@ -172,6 +196,10 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { } catch (e : IOException){ Log.d(TAG, "testConnection [connect]: ${e.message}") return false + } catch (e : SecurityException) + { + Log.d(TAG, "testConnection [connect]: ${e.message}") + return false } var isValid = false runBlocking(Dispatchers.IO) { @@ -215,8 +243,15 @@ class BluetoothStreamer(private val ctx : Context) : Streamer { // get connected device information override fun getInfo() : String { - if(adapter == null || target == null || socket == null) return "" - return "[Device Name] ${target?.name}\n[Device Address] ${target?.address}" + if(target == null || socket == null) return "" + val deviceName = try { + target?.name + } catch (e : SecurityException) + { + Log.d(TAG, "getInfo: ${e.message}") + "null" + } + return "[Device Name] ${deviceName}\n[Device Address] ${target?.address}" } // return true if is connected for streaming diff --git a/Android/build.gradle b/Android/build.gradle index c348a6f..02ae24c 100644 --- a/Android/build.gradle +++ b/Android/build.gradle @@ -6,7 +6,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.0.0' + classpath 'com.android.tools.build:gradle:7.1.3' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" // NOTE: Do not place your application dependencies here; they belong diff --git a/Android/gradle/wrapper/gradle-wrapper.properties b/Android/gradle/wrapper/gradle-wrapper.properties index 0e6615d..d1a373c 100644 --- a/Android/gradle/wrapper/gradle-wrapper.properties +++ b/Android/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon May 24 15:46:59 EDT 2021 distributionBase=GRADLE_USER_HOME -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip distributionPath=wrapper/dists zipStorePath=wrapper/dists zipStoreBase=GRADLE_USER_HOME diff --git a/Assets/p1.png b/Assets/p1.png index 8de693b..b139d33 100644 Binary files a/Assets/p1.png and b/Assets/p1.png differ diff --git a/Assets/p4.png b/Assets/p4.png new file mode 100644 index 0000000..68c97c6 Binary files /dev/null and b/Assets/p4.png differ diff --git a/Assets/sound_config6.png b/Assets/sound_config6.png new file mode 100644 index 0000000..0bf1723 Binary files /dev/null and b/Assets/sound_config6.png differ diff --git a/README.md b/README.md index ecba38b..212b0f1 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,11 @@ Use your Android phone as a microphone to Windows PC 6. For microphone, click `Properties` and set following: sound config5 -On my machine, this setup has the lowest delay and best sound quality. VB is not optimized as hardware devices, so these configurations are important for audio. +On my machine, this setup has the lowest delay and best sound quality. +Can further improve audio latency by opening `VBCABLE_ControlPanel.exe` (from downloaded folder of VB) and set `Max Latency` in Options to 2048 smp: +sound config6 + +Do not set to 512 smp since that will cause most buffers lost. If 2048 has no sound or bad quality, consider a higher smp. @@ -106,7 +110,7 @@ __SpeexDSP Filters__: - [ ] Audio effect filters (Not yet released) - [x] Pitch Shifter - [x] Add White Noise - - [x] Repeated Background Track + - [x] Repeated Background Audio - [x] SpeexDSP Noise Cancellation - [x] SpeexDSP Automatic Gain Control - [x] SpeexDSP Voice Activity Detection @@ -121,18 +125,18 @@ Check out [Avalonia](https://github.com/AvaloniaUI/Avalonia)! With this it may b Pre-built installers can be found [here](https://github.com/teamclouday/AndroidMic/releases) - ------ ## Windows Side -Windows Side - -## Android Side (Portrait) +

+Windows Side +Windows Side +

-Android Side +## Android Side -## Android Side (Landscape) +Android Side Android Side diff --git a/Windows/AndroidMic/AdvancedWindow.xaml b/Windows/AndroidMic/AdvancedWindow.xaml index 5db2501..1c8675a 100644 --- a/Windows/AndroidMic/AdvancedWindow.xaml +++ b/Windows/AndroidMic/AdvancedWindow.xaml @@ -66,7 +66,7 @@ - AndroidMic Teamclouday 0 - 1.6.0.0 + 1.7.0.0 false true true diff --git a/Windows/AndroidMic/Library/Audio/AudioBuffer.cs b/Windows/AndroidMic/Library/Audio/AudioBuffer.cs index b423bbf..cf6d1fa 100644 --- a/Windows/AndroidMic/Library/Audio/AudioBuffer.cs +++ b/Windows/AndroidMic/Library/Audio/AudioBuffer.cs @@ -4,7 +4,7 @@ namespace AndroidMic.Audio { public class AudioBuffer { - public const int MAX_BUFFER_SIZE = 2; + public const int MAX_BUFFER_SIZE = 3; private readonly Queue buffer = new Queue(); private readonly object toLock = new object(); diff --git a/Windows/AndroidMic/Library/Audio/Filters/FilterSpeexDSP.cs b/Windows/AndroidMic/Library/Audio/Filters/FilterSpeexDSP.cs index c3dd26c..2ca2465 100644 --- a/Windows/AndroidMic/Library/Audio/Filters/FilterSpeexDSP.cs +++ b/Windows/AndroidMic/Library/Audio/Filters/FilterSpeexDSP.cs @@ -52,6 +52,7 @@ public enum ConfigTypes private Ellipse indicator; private readonly Brush indicatorOn; private readonly Brush indicatorOff; + private bool indicatorInSpeech; private readonly IntPtr SpeexPreprocessState; private readonly IntPtr SpeexEchoState; @@ -112,7 +113,7 @@ public int Read(byte[] buffer, int offset, int sampleCount) Buffer.BlockCopy(echoOutBuffer, 0, audioBuffer, 0, nextRead); } // process audio - InSpeech = SpeexPreprocess.speex_preprocess_run(SpeexPreprocessState, audioBuffer) == 1; + indicatorInSpeech = SpeexPreprocess.speex_preprocess_run(SpeexPreprocessState, audioBuffer) == 1; // copy back Buffer.BlockCopy(audioBuffer, 0, buffer, offset, nextRead); // update samples to read @@ -120,7 +121,7 @@ public int Read(byte[] buffer, int offset, int sampleCount) offset += nextRead; } // check if VAD enabled - InSpeech = InSpeech && EnabledVAD; + indicatorInSpeech = indicatorInSpeech && EnabledVAD; // update indicator Application.Current.Dispatcher.Invoke(new Action(() => { @@ -148,7 +149,7 @@ public void SetIndicator(Ellipse e) private void UpdateIndicator() { if (indicator == null) return; - indicator.Fill = InSpeech ? indicatorOn : indicatorOff; + indicator.Fill = indicatorInSpeech ? indicatorOn : indicatorOff; } // start PC speaker capture @@ -296,6 +297,15 @@ public bool EnabledVAD SpeexPreprocessState, SpeexPreprocess.SPEEX_PREPROCESS_SET_PROB_CONTINUE, StateUpdate); } + else + { + indicatorInSpeech = false; + // update indicator + Application.Current.Dispatcher.Invoke(new Action(() => + { + UpdateIndicator(); + })); + } } } @@ -337,7 +347,5 @@ public bool EnabledEcho else StopCapture(); } } - - public bool InSpeech { get; private set; } } } diff --git a/Windows/README.md b/Windows/README.md index d9d4852..c262e80 100644 --- a/Windows/README.md +++ b/Windows/README.md @@ -38,14 +38,15 @@ Built with WPF I once thought TCP delays audio data transfer. So I went to look at RTSP as alternative. Turns out it uses TCP for control and UDP/TCP to transfer data. RTSP is mainly used to stream video (with audio) data. TCP is still fast enough for audio data. No stream control required in my app, so RTSP is not necessary. BTW, UDP transfer is not easy to implement without a sequential order manager. -### Buffers +### Latency -A lot of buffers used in one audio pass: -1. Android `AudioRecord` will first store audio data in a buffer -2. Then recorded data will be copied to `AudioBuffer` by audio manager, which will be read by stream manager -3. Stream manager reads and sends to Windows side. Windows stream manager receives and stores in `AudioBuffer`, which will be read by audio manager -4. Audio manager has a `BufferedWaveProvider` layer for `NAudio` player, which will also store cached audio data +By test, audio via bluetooth socket has much __higher latency__ than TCP socket through Wifi on my machine, even though the code is very similar. +VB Cable can be configured to minimum latency, but still slower than physical devices. +Android `AudioRecord` also has latency for recording. Can look into [Oboe](https://github.com/google/oboe) to replace `AudioRecord` to improve. -The size and implementation of these buffers affect the latency of audio transfer. Two `AudioBuffer`s can be configured. I set max number of buffers to `3`. (Can also try 2 but it may drop audio too frequently) For `NAudio` wave out player, I set desired latency to `50`, number of buffers to `3`. A combination of (50,2) will cause choppy audio. -Assume Android `AudioRecord` has optimum performance. Audio format is `16000` sample rate, `16` bits (2 bytes) data, `1` channel, PCM. Expect to have 16000x2x1=`32000` bytes generated from Android app per second. Bluetooth is able to transfer at least 1Mbps. Wifi will be much faster (up to 2 Gbps). TCP header can be up to 60 bytes per packet. So sockets are not expected to cause delay. \ No newline at end of file +### SpeexDSP and Audio Processing + +This projects integrates [SpeexDSP](https://gitlab.xiph.org/xiph/speexdsp) library to support echo cancellation, noise suppression, automatic gain control and voice activity detection. The library dll provided in this project is compiled locally from the [latest source code](https://gitlab.xiph.org/xiph/speexdsp/-/commit/68311d46785be76d2a186c75578d51108bff6dfb) for x86. Then I wrote a C# binding for the library to call the functions in header files. +It is tricky to configure SpeexDSP filter, especially for echo cancellation. I looked at a lot of open source projects to learn their parameters. I tweaked and tested those parameters to better fit my app. For echo cancellation, a player buffer is required, which should be the samples sent to the speaker. I use NAudio provider which calls WASAPI internally to record the PC soundcard output. +Alternative for audio processing is [WebRTC](https://webrtc.org/). It is huge library in comparison, so I didn't include it in the end. \ No newline at end of file