How to get microphone data?

[paulocoutinhox]

Hi,

How to get microphone data?

Like volume and frequency.

Thanks.

[paulocoutinhox]

Simple code:

SWIFT

import Foundation

class AudioCapture: NSObject {
    var audioEngine: AVAudioEngine!
    let audioBridge = AudioBridge()  // Instância da ponte Objective-C++
    
    func startRecording() {
        audioEngine = AVAudioEngine()
        let inputNode = audioEngine.inputNode
        let recordingFormat = inputNode.outputFormat(forBus: 0)

        inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { (buffer, time) in
            let channelData = buffer.floatChannelData?[0]
            let channelDataValueArray = stride(from: 0, to: Int(buffer.frameLength), by: buffer.stride).map { channelData![$0] }

            let volume = self.calculateVolume(buffer: channelDataValueArray)
            let audioBuffer = self.convertToInt16(buffer: channelDataValueArray)
            // Agora chama o método da ponte que envia os dados para o C++
            self.audioBridge.processAudioBuffer(audioBuffer, length: Int32(audioBuffer.count), volume: Int32(volume))
        }

        try? AVAudioSession.sharedInstance().setCategory(.record, mode: .measurement, options: .duckOthers)
        try? AVAudioSession.sharedInstance().setActive(true)
        audioEngine.prepare()
        try? audioEngine.start()
    }

    private func calculateVolume(buffer: [Float]) -> Float {
        let rms = sqrt(buffer.map { $0 * $0 }.reduce(0, +) / Float(buffer.count))
        let volume = max(0, min(100, 20 * log10(rms) + 100)) // Volume normalizado 0 a 100
        return volume
    }

    private func convertToInt16(buffer: [Float]) -> [Int16] {
        return buffer.map { Int16($0 * Float(Int16.max)) }
    }

    func stopRecording() {
        audioEngine.stop()
        audioEngine.inputNode.removeTap(onBus: 0)
    }
}

OBJC

#import <Foundation/Foundation.h>

@interface AudioBridge : NSObject

- (void)processAudioBuffer:(int16_t *)buffer length:(int32_t)length volume:(int32_t)volume;

@end

#import "AudioBridge.h"
#include "AudioProcessor.h"  // Inclua seu arquivo de cabeçalho C++ aqui

@implementation AudioBridge

- (void)processAudioBuffer:(int16_t *)buffer length:(int32_t)length volume:(int32_t)volume {
    // Chama a função C++ para processar o buffer de áudio
    processAudioBuffer(buffer, length, volume);
}

@end

C - JNI

#include <jni.h>
#include "AudioProcessor.h"

extern "C"
JNIEXPORT void JNICALL
Java_com_mygame_AudioCapture_processAudio(JNIEnv *env, jobject thiz, jshortArray buffer, jint length, jint volume) {
    jshort *audioData = env->GetShortArrayElements(buffer, NULL);

    // Processar os dados de áudio no C++
    processAudioBuffer(audioData, length, volume);

    env->ReleaseShortArrayElements(buffer, audioData, 0);
}

JAVA

import android.Manifest;
import android.app.Activity;
import android.content.pm.PackageManager;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Build;
import android.util.Log;
import androidx.core.app.ActivityCompat;

public class AudioCapture {
    private AudioRecord recorder;
    private int bufferSize;
    private short[] buffer;
    private boolean isRecording = false;
    private Thread recordingThread;

    private static final int REQUEST_MICROPHONE = 1;

    public void requestMicrophonePermission(Activity activity) {
        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
            if (activity.checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
                ActivityCompat.requestPermissions(activity, new String[]{Manifest.permission.RECORD_AUDIO}, REQUEST_MICROPHONE);
            }
        }
    }

    public void startRecording() {
        int sampleRate = 44100;
        bufferSize = AudioRecord.getMinBufferSize(sampleRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT);
        recorder = new AudioRecord(MediaRecorder.AudioSource.MIC, sampleRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize);

        buffer = new short[bufferSize];
        recorder.startRecording();
        isRecording = true;

        recordingThread = new Thread(() -> {
            while (isRecording) {
                int read = recorder.read(buffer, 0, buffer.length);
                if (read > 0) {
                    int volume = calculateVolume(buffer, read);
                    // Enviar os dados para o C++ via JNI
                    processAudio(buffer, read, volume);
                }
            }
        });
        recordingThread.start();
    }

    public void stopRecording() {
        if (recorder != null) {
            isRecording = false;
            recorder.stop();
            recorder.release();
            recorder = null;
            recordingThread = null;
        }
    }

    private int calculateVolume(short[] audioData, int length) {
        double sum = 0;
        for (int i = 0; i < length; i++) {
            sum += audioData[i] * audioData[i];
        }
        double rms = Math.sqrt(sum / length);
        return (int) Math.max(0, Math.min(100, 20 * Math.log10(rms)));
    }

    public native void processAudio(short[] buffer, int length, int volume);
}

C++ (volume and note)

#include "kiss_fft.h"
#include <math.h>
#include <iostream>

void processAudioBuffer(int16_t* audioData, int length, int volume) {
    // Volume já normalizado de 0 a 100
    int normalizedVolume = std::max(0, std::min(100, volume));

    // Configuração da FFT
    int nfft = 1024;
    kiss_fft_cfg cfg = kiss_fft_alloc(nfft, 0, NULL, NULL);
    kiss_fft_cpx in[nfft], out[nfft];

    // Preenchendo os dados da FFT
    for (int i = 0; i < nfft && i < length; i++) {
        in[i].r = audioData[i];
        in[i].i = 0;
    }

    // Executando a FFT
    kiss_fft(cfg, in, out);

    // Encontrando a frequência dominante
    double maxAmplitude = 0;
    int dominantIndex = 0;
    for (int i = 0; i < nfft / 2; i++) {
        double amplitude = sqrt(out[i].r * out[i].r + out[i].i * out[i].i);
        if (amplitude > maxAmplitude) {
            maxAmplitude = amplitude;
            dominantIndex = i;
        }
    }

    // Convertendo índice FFT para frequência
    double frequency = dominantIndex * 44100.0 / nfft;  // Supondo taxa de amostragem de 44100Hz

    // Mapeando frequência para notas musicais
    double noteValue = round(12 * log2(frequency / 440.0)) + 69;  // A4 (440Hz) como referência
    char note = mapFrequencyToNote(noteValue);

    // Exibindo o volume e a nota
    std::cout << "Volume: " << normalizedVolume << "/100, Nota: " << note << std::endl;

    kiss_fft_free(cfg);
}

char mapFrequencyToNote(double noteValue) {
    int note = static_cast<int>(noteValue) % 12;
    switch (note) {
        case 0: return 'A';
        case 1: return 'A';
        case 2: return 'B';
        case 3: return 'C';
        case 4: return 'C';
        case 5: return 'D';
        case 6: return 'D';
        case 7: return 'E';
        case 8: return 'F';
        case 9: return 'F';
        case 10: return 'G';
        case 11: return 'G';
        default: return '?';
    }
}

Maybe help someone.

[aismann]

Thank you! I hope I can use it sometime.
@paulocoutinhox
Can you switch it to Q&A plz?

[paulocoutinhox]

done

[johnspeny]

I believe its also good for those games where you shout to move objects in the game. Nice work.

[aismann]

done

@paulocoutinhox
I cant mark the first entry as answer.
Can you do it plz?

[aismann]

I believe its also good for those games where you shout to move objects in the game. Nice work.

A flappy bird controlled with voice is a nice idea