Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[HTML5] Move audio processing to thread when threads are enabled. #42505

Merged
merged 3 commits into from
Oct 2, 2020
Merged

[HTML5] Move audio processing to thread when threads are enabled. #42505

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5261e5d
Add COOP/COEP headers to HTML5 "run" server.
Faless Sep 30, 2020
80b34cc
Add extra suffix for HTML5 thread builds.
Faless Sep 30, 2020
a618535
[HTML5] Run Audio process in thread when available
Faless Oct 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions platform/javascript/SCsub
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ build = env.add_program(build_targets, javascript_files)

js_libraries = [
"native/http_request.js",
"native/library_godot_audio.js",
]
for lib in js_libraries:
env.Append(LINKFLAGS=["--js-library", env.File(lib).path])
Expand Down
248 changes: 65 additions & 183 deletions platform/javascript/audio_driver_javascript.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,34 +31,57 @@
#include "audio_driver_javascript.h"

#include "core/project_settings.h"
#include "godot_audio.h"

#include <emscripten.h>

AudioDriverJavaScript *AudioDriverJavaScript::singleton = nullptr;

bool AudioDriverJavaScript::is_available() {
return EM_ASM_INT({
if (!(window.AudioContext || window.webkitAudioContext)) {
return 0;
}
return 1;
}) != 0;
return godot_audio_is_available() != 0;
}

const char *AudioDriverJavaScript::get_name() const {
return "JavaScript";
}

extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_js_mix() {
AudioDriverJavaScript::singleton->mix_to_js();
#ifndef NO_THREADS
void AudioDriverJavaScript::_audio_thread_func(void *p_data) {
AudioDriverJavaScript *obj = static_cast<AudioDriverJavaScript *>(p_data);
while (!obj->quit) {
obj->lock();
if (!obj->needs_process) {
obj->unlock();
OS::get_singleton()->delay_usec(1000); // Give the browser some slack.
continue;
}
obj->_js_driver_process();
obj->needs_process = false;
obj->unlock();
}
}
#endif

extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_start() {
#ifndef NO_THREADS
AudioDriverJavaScript::singleton->lock();
#else
AudioDriverJavaScript::singleton->_js_driver_process();
#endif
}

extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_end() {
#ifndef NO_THREADS
AudioDriverJavaScript::singleton->needs_process = true;
AudioDriverJavaScript::singleton->unlock();
#endif
}

extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_capture(float sample) {
AudioDriverJavaScript::singleton->process_capture(sample);
}

void AudioDriverJavaScript::mix_to_js() {
int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
void AudioDriverJavaScript::_js_driver_process() {
int sample_count = memarr_len(internal_buffer) / channel_count;
int32_t *stream_buffer = reinterpret_cast<int32_t *>(internal_buffer);
audio_server_process(sample_count, stream_buffer);
Expand All @@ -73,37 +96,12 @@ void AudioDriverJavaScript::process_capture(float sample) {
}

Error AudioDriverJavaScript::init() {
int mix_rate = GLOBAL_GET("audio/mix_rate");
mix_rate = GLOBAL_GET("audio/mix_rate");
int latency = GLOBAL_GET("audio/output_latency");

/* clang-format off */
_driver_id = EM_ASM_INT({
const MIX_RATE = $0;
const LATENCY = $1 / 1000;
return Module.IDHandler.add({
'context': new (window.AudioContext || window.webkitAudioContext)({ sampleRate: MIX_RATE, latencyHint: LATENCY}),
'input': null,
'stream': null,
'script': null
});
}, mix_rate, latency);
/* clang-format on */

int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
channel_count = godot_audio_init(mix_rate, latency);
buffer_length = closest_power_of_2((latency * mix_rate / 1000) * channel_count);
/* clang-format off */
buffer_length = EM_ASM_INT({
var ref = Module.IDHandler.get($0);
const ctx = ref['context'];
const BUFFER_LENGTH = $1;
const CHANNEL_COUNT = $2;

var script = ctx.createScriptProcessor(BUFFER_LENGTH, 2, CHANNEL_COUNT);
script.connect(ctx.destination);
ref['script'] = script;
return script.bufferSize;
}, _driver_id, buffer_length, channel_count);
/* clang-format on */
buffer_length = godot_audio_create_processor(buffer_length, channel_count);
if (!buffer_length) {
return FAILED;
}
Expand All @@ -114,191 +112,75 @@ Error AudioDriverJavaScript::init() {
internal_buffer = memnew_arr(float, buffer_length *channel_count);
}

return internal_buffer ? OK : ERR_OUT_OF_MEMORY;
if (!internal_buffer) {
return ERR_OUT_OF_MEMORY;
}
return OK;
}

void AudioDriverJavaScript::start() {
/* clang-format off */
EM_ASM({
const ref = Module.IDHandler.get($0);
var INTERNAL_BUFFER_PTR = $1;

var audioDriverMixFunction = cwrap('audio_driver_js_mix');
var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
ref['script'].onaudioprocess = function(audioProcessingEvent) {
audioDriverMixFunction();

var input = audioProcessingEvent.inputBuffer;
var output = audioProcessingEvent.outputBuffer;
var internalBuffer = HEAPF32.subarray(
INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT,
INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);

for (var channel = 0; channel < output.numberOfChannels; channel++) {
var outputData = output.getChannelData(channel);
// Loop through samples.
for (var sample = 0; sample < outputData.length; sample++) {
outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
}
}

if (ref['input']) {
var inputDataL = input.getChannelData(0);
var inputDataR = input.getChannelData(1);
for (var i = 0; i < inputDataL.length; i++) {
audioDriverProcessCapture(inputDataL[i]);
audioDriverProcessCapture(inputDataR[i]);
}
}
};
}, _driver_id, internal_buffer);
/* clang-format on */
#ifndef NO_THREADS
thread = Thread::create(_audio_thread_func, this);
#endif
godot_audio_start(internal_buffer);
}

void AudioDriverJavaScript::resume() {
/* clang-format off */
EM_ASM({
const ref = Module.IDHandler.get($0);
if (ref && ref['context'] && ref['context'].resume)
ref['context'].resume();
}, _driver_id);
/* clang-format on */
godot_audio_resume();
}

float AudioDriverJavaScript::get_latency() {
/* clang-format off */
return EM_ASM_DOUBLE({
const ref = Module.IDHandler.get($0);
var latency = 0;
if (ref && ref['context']) {
const ctx = ref['context'];
if (ctx.baseLatency) {
latency += ctx.baseLatency;
}
if (ctx.outputLatency) {
latency += ctx.outputLatency;
}
}
return latency;
}, _driver_id);
/* clang-format on */
return godot_audio_get_latency();
}

int AudioDriverJavaScript::get_mix_rate() const {
/* clang-format off */
return EM_ASM_INT({
const ref = Module.IDHandler.get($0);
return ref && ref['context'] ? ref['context'].sampleRate : 0;
}, _driver_id);
/* clang-format on */
return mix_rate;
}

AudioDriver::SpeakerMode AudioDriverJavaScript::get_speaker_mode() const {
/* clang-format off */
return get_speaker_mode_by_total_channels(EM_ASM_INT({
const ref = Module.IDHandler.get($0);
return ref && ref['context'] ? ref['context'].destination.channelCount : 0;
}, _driver_id));
/* clang-format on */
return get_speaker_mode_by_total_channels(channel_count);
}

// No locking, as threads are not supported.
void AudioDriverJavaScript::lock() {
#ifndef NO_THREADS
mutex.lock();
#endif
}

void AudioDriverJavaScript::unlock() {
#ifndef NO_THREADS
mutex.unlock();
#endif
}

void AudioDriverJavaScript::finish_async() {
// Close the context, add the operation to the async_finish list in module.
int id = _driver_id;
_driver_id = 0;

/* clang-format off */
EM_ASM({
const id = $0;
var ref = Module.IDHandler.get(id);
Module.async_finish.push(new Promise(function(accept, reject) {
if (!ref) {
console.log("Ref not found!", id, Module.IDHandler);
setTimeout(accept, 0);
} else {
Module.IDHandler.remove(id);
const context = ref['context'];
// Disconnect script and input.
ref['script'].disconnect();
if (ref['input'])
ref['input'].disconnect();
ref = null;
context.close().then(function() {
accept();
}).catch(function(e) {
accept();
});
}
}));
}, id);
/* clang-format on */
#ifndef NO_THREADS
quit = true; // Ask thread to quit.
#endif
godot_audio_finish_async();
}

void AudioDriverJavaScript::finish() {
#ifndef NO_THREADS
Thread::wait_to_finish(thread);
memdelete(thread);
thread = NULL;
#endif
if (internal_buffer) {
memdelete_arr(internal_buffer);
internal_buffer = nullptr;
}
}

Error AudioDriverJavaScript::capture_start() {
godot_audio_capture_stop();
input_buffer_init(buffer_length);

/* clang-format off */
EM_ASM({
function gotMediaInput(stream) {
var ref = Module.IDHandler.get($0);
ref['stream'] = stream;
ref['input'] = ref['context'].createMediaStreamSource(stream);
ref['input'].connect(ref['script']);
}

function gotMediaInputError(e) {
out(e);
}

if (navigator.mediaDevices.getUserMedia) {
navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
} else {
if (!navigator.getUserMedia)
navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
}
}, _driver_id);
/* clang-format on */

godot_audio_capture_start();
return OK;
}

Error AudioDriverJavaScript::capture_stop() {
/* clang-format off */
EM_ASM({
var ref = Module.IDHandler.get($0);
if (ref['stream']) {
const tracks = ref['stream'].getTracks();
for (var i = 0; i < tracks.length; i++) {
tracks[i].stop();
}
ref['stream'] = null;
}

if (ref['input']) {
ref['input'].disconnect();
ref['input'] = null;
}

}, _driver_id);
/* clang-format on */

input_buffer.clear();

return OK;
}

Expand Down
19 changes: 17 additions & 2 deletions platform/javascript/audio_driver_javascript.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,30 @@

#include "servers/audio_server.h"

#include "core/os/mutex.h"
#include "core/os/thread.h"

class AudioDriverJavaScript : public AudioDriver {
private:
float *internal_buffer = nullptr;

int _driver_id = 0;
int buffer_length = 0;
int mix_rate = 0;
int channel_count = 0;

public:
#ifndef NO_THREADS
Mutex mutex;
Thread *thread = nullptr;
bool quit = false;
bool needs_process = true;

static void _audio_thread_func(void *p_data);
#endif

void _js_driver_process();

static bool is_available();
void mix_to_js();
void process_capture(float sample);

static AudioDriverJavaScript *singleton;
Expand Down
1 change: 1 addition & 0 deletions platform/javascript/detect.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ def configure(env):
env.Append(LINKFLAGS=["-s", "USE_PTHREADS=1"])
env.Append(LINKFLAGS=["-s", "PTHREAD_POOL_SIZE=4"])
env.Append(LINKFLAGS=["-s", "WASM_MEM_MAX=2048MB"])
env.extra_suffix = ".threads" + env.extra_suffix
else:
env.Append(CPPDEFINES=["NO_THREADS"])

Expand Down
3 changes: 3 additions & 0 deletions platform/javascript/export/export.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,9 @@ class EditorHTTPServer : public Reference {
String s = "HTTP/1.1 200 OK\r\n";
s += "Connection: Close\r\n";
s += "Content-Type: " + ctype + "\r\n";
s += "Access-Control-Allow-Origin: *\r\n";
s += "Cross-Origin-Opener-Policy: same-origin\r\n";
s += "Cross-Origin-Embedder-Policy: require-corp\r\n";
s += "\r\n";
CharString cs = s.utf8();
Error err = connection->put_data((const uint8_t *)cs.get_data(), cs.size() - 1);
Expand Down
Loading