From e213f561790cc0fa94c7654232dd4e1fc602df97 Mon Sep 17 00:00:00 2001 From: Julian Todd Date: Wed, 18 Dec 2024 00:29:53 +0000 Subject: [PATCH] using new hacked godot source code from https://github.com/godotengine/godot/pull/100508 --- example/default_bus_layout.tres | 9 +++- example/minimalusecase/minimalusecase.gd | 10 +++++ example/minimalusecase/minimalusecase.tscn | 2 +- example/project.godot | 2 +- example/radiomqtt/member.tscn | 2 +- example/radiomqtt/radiomqtt.gd | 25 ++++++++--- example/radiomqtt/radiomqtt.tscn | 13 +++--- src/audio_effect_opus_chunked.cpp | 52 ++++++++++++++++++++++ src/audio_effect_opus_chunked.h | 16 +++++++ 9 files changed, 117 insertions(+), 14 deletions(-) diff --git a/example/default_bus_layout.tres b/example/default_bus_layout.tres index 3335e8f..94bc674 100644 --- a/example/default_bus_layout.tres +++ b/example/default_bus_layout.tres @@ -1,8 +1,13 @@ -[gd_resource type="AudioBusLayout" load_steps=2 format=3 uid="uid://orx4iw038t0c"] +[gd_resource type="AudioBusLayout" load_steps=3 format=3 uid="uid://orx4iw038t0c"] [sub_resource type="AudioEffectPitchShift" id="AudioEffectPitchShift_urnej"] resource_name = "PitchShift" +[sub_resource type="AudioEffectCapture" id="AudioEffectCapture_sj64h"] +resource_local_to_scene = true +resource_name = "Capture" +buffer_length = 0.3 + [resource] bus/1/name = &"SpeechBus" bus/1/solo = false @@ -18,3 +23,5 @@ bus/2/mute = true bus/2/bypass_fx = false bus/2/volume_db = 0.0 bus/2/send = &"Master" +bus/2/effect/0/effect = SubResource("AudioEffectCapture_sj64h") +bus/2/effect/0/enabled = true diff --git a/example/minimalusecase/minimalusecase.gd b/example/minimalusecase/minimalusecase.gd index 3d5ebd9..16bed11 100644 --- a/example/minimalusecase/minimalusecase.gd +++ b/example/minimalusecase/minimalusecase.gd @@ -5,6 +5,8 @@ var audiostreamopuschunked : AudioStreamOpusChunked var prepend = PackedByteArray() var opuspacketsbuffer = [ ] +@onready var playbackmic = AudioStreamPlaybackMicrophone.new() + func _ready(): var microphoneidx = AudioServer.get_bus_index("MicrophoneBus") opuschunked = AudioServer.get_bus_effect(microphoneidx, 0) @@ -13,10 +15,18 @@ func _ready(): for r in opusaudiodata: opuspacketsbuffer.append(PackedByteArray(r)) + playbackmic.start(0.0) + func _process(delta): #_process_record(delta) _process_playback(delta) +func _input(event): + if event is InputEventKey and event.pressed and event.keycode == KEY_D: + print(playbackmic.is_playing()) + print(playbackmic.get_microphone_buffer(20)+playbackmic.get_microphone_buffer(20)) + + func _process_record(delta): while opuschunked.chunk_available(): var chunkmax = opuschunked.chunk_max(false, false) diff --git a/example/minimalusecase/minimalusecase.tscn b/example/minimalusecase/minimalusecase.tscn index 3c5f948..51ef12d 100644 --- a/example/minimalusecase/minimalusecase.tscn +++ b/example/minimalusecase/minimalusecase.tscn @@ -1,6 +1,6 @@ [gd_scene load_steps=4 format=3 uid="uid://cx10t585wbeo2"] -[ext_resource type="Script" path="res://minimalusecase/minimalusecase.gd" id="1_m25c5"] +[ext_resource type="Script" uid="uid://dg10tsmim56bv" path="res://minimalusecase/minimalusecase.gd" id="1_m25c5"] [sub_resource type="AudioStreamMicrophone" id="AudioStreamMicrophone_c8xp2"] diff --git a/example/project.godot b/example/project.godot index f7515ec..cd9ee13 100644 --- a/example/project.godot +++ b/example/project.godot @@ -12,7 +12,7 @@ config_version=5 config/name="Two-voip-godot-4" run/main_scene="res://radiomqtt/radiomqtt.tscn" -config/features=PackedStringArray("4.3", "GL Compatibility") +config/features=PackedStringArray("4.4", "GL Compatibility") config/icon="res://icon.png" [audio] diff --git a/example/radiomqtt/member.tscn b/example/radiomqtt/member.tscn index 01129d8..fa6b8ed 100644 --- a/example/radiomqtt/member.tscn +++ b/example/radiomqtt/member.tscn @@ -1,6 +1,6 @@ [gd_scene load_steps=6 format=3 uid="uid://c1ibcb1toyerh"] -[ext_resource type="Script" path="res://radiomqtt/member.gd" id="1_o3hks"] +[ext_resource type="Script" uid="uid://bckl548pomyik" path="res://radiomqtt/member.gd" id="1_o3hks"] [ext_resource type="Texture2D" uid="uid://cvyjsyigutxbh" path="res://icon.png" id="2_atirx"] [sub_resource type="AudioStreamGenerator" id="AudioStreamGenerator_t0bqw"] diff --git a/example/radiomqtt/radiomqtt.gd b/example/radiomqtt/radiomqtt.gd index 885aabd..48a9f62 100644 --- a/example/radiomqtt/radiomqtt.gd +++ b/example/radiomqtt/radiomqtt.gd @@ -9,6 +9,7 @@ var audioeffectpitchshift : AudioEffectPitchShift = null var audioeffectpitchshiftidx = 0 var audioeffectcapture : AudioEffectCapture = null var audioopuschunkedeffect # : AudioEffectOpusChunked +@onready var audiostreamplaybackmicrophone : AudioStreamPlaybackMicrophone = AudioStreamPlaybackMicrophone.new() # values to use when AudioEffectOpusChunked cannot be instantiated var frametimems : float = 20 @@ -44,6 +45,8 @@ var visemes = [ "sil", "PP", "FF", "TH", "DD", "kk", "CH", "SS", "nn", "RR", "aa var possibleusernames = ["Alice", "Beth", "Cath", "Dan", "Earl", "Fred", "George", "Harry", "Ivan", "John", "Kevin", "Larry", "Martin", "Oliver", "Peter", "Quentin", "Robert", "Samuel", "Thomas", "Ulrik", "Victor", "Wayne", "Xavier", "Youngs", "Zephir"] func _ready(): + if audiostreamplaybackmicrophone != null: + audiostreamplaybackmicrophone.start_microphone() print("AudioServer.get_mix_rate()=", AudioServer.get_mix_rate()) print("ProjectSettings.get_setting_with_override(\"audio/driver/mix_rate\")=", ProjectSettings.get_setting_with_override("audio/driver/mix_rate")) $VBoxPlayback/HBoxStream/MixRate.value = AudioServer.get_mix_rate() @@ -289,7 +292,8 @@ func _input(event): $AudioStreamMicrophone.volume_db += (1 if event.keycode == KEY_I else -1) print($AudioStreamMicrophone.volume_db) - + if event.keycode == KEY_D: + audioopuschunkedeffect.Drunmicthing() func _process(_delta): @@ -299,14 +303,14 @@ func _process(_delta): if talking: $VBoxPlayback/HBoxPlaycount/GridContainer/TimeSecs.text = "%.1f" % ((Time.get_ticks_msec() - talkingstarttime)*0.001) if talking and not currentlytalking: - if not $AudioStreamMicrophone.playing: + if not $AudioStreamMicrophone.playing and audiostreamplaybackmicrophone == null: $AudioStreamMicrophone.play() starttalking() elif not talking and currentlytalking: endtalking() $HBoxMicTalk/MicWorking.button_pressed = $AudioStreamMicrophone.playing - if audioeffectcapture == null: + if audioeffectcapture == null and audiostreamplaybackmicrophone == null: assert (audioopuschunkedeffect != null) while audioopuschunkedeffect.chunk_available(): var audiosamples = audioopuschunkedeffect.read_chunk(false) @@ -357,8 +361,19 @@ func _process(_delta): audioopuschunkedeffect.drop_chunk() else: - while audioeffectcapture.get_frames_available() > audiosamplesize: - var audiosamples = audioeffectcapture.get_buffer(audiosamplesize) + while true: + var audiosamples = null + if audiostreamplaybackmicrophone.is_microphone_playing() and audiostreamplaybackmicrophone != null: + audiosamples = audiostreamplaybackmicrophone.get_microphone_buffer(audiosamplesize) + if len(audiosamples) != audiosamplesize: + assert (len(audiosamples) == 0) + break + else: + if audioeffectcapture.get_frames_available() > audiosamplesize: + audiosamples = audioeffectcapture.get_buffer(audiosamplesize) + else: + break + audiosamplestoshader(audiosamples, false) var chunkv1 = 0.0 var schunkv2 = 0.0 diff --git a/example/radiomqtt/radiomqtt.tscn b/example/radiomqtt/radiomqtt.tscn index 7413c3d..1b82a95 100644 --- a/example/radiomqtt/radiomqtt.tscn +++ b/example/radiomqtt/radiomqtt.tscn @@ -1,10 +1,10 @@ [gd_scene load_steps=22 format=3 uid="uid://babntosbiq5r8"] -[ext_resource type="Script" path="res://radiomqtt/radiomqtt.gd" id="1_anot2"] +[ext_resource type="Script" uid="uid://bdckem1ncyrbm" path="res://radiomqtt/radiomqtt.gd" id="1_anot2"] [ext_resource type="PackedScene" uid="uid://ktm7k0co2o7l" path="res://addons/mqtt/mqtt.tscn" id="2_4wm5t"] -[ext_resource type="Script" path="res://radiomqtt/MQTTnetwork.gd" id="2_y485j"] +[ext_resource type="Script" uid="uid://bnox12o7ro1j6" path="res://radiomqtt/MQTTnetwork.gd" id="2_y485j"] [ext_resource type="Texture2D" uid="uid://cvyjsyigutxbh" path="res://icon.png" id="3_eh2h5"] -[ext_resource type="Script" path="res://radiomqtt/voxdetector.gd" id="3_qvq17"] +[ext_resource type="Script" uid="uid://byx6ipd6c6ldh" path="res://radiomqtt/voxdetector.gd" id="3_qvq17"] [ext_resource type="PackedScene" uid="uid://c1ibcb1toyerh" path="res://radiomqtt/member.tscn" id="4_w5dv2"] [sub_resource type="AudioStreamMicrophone" id="AudioStreamMicrophone_itmlg"] @@ -57,8 +57,8 @@ void fragment() { [sub_resource type="ShaderMaterial" id="ShaderMaterial_cklni"] shader = SubResource("Shader_r847e") -shader_parameter/drawresampled = false shader_parameter/voice = ExtResource("3_eh2h5") +shader_parameter/drawresampled = false shader_parameter/voice_resampled = ExtResource("3_eh2h5") [sub_resource type="StyleBoxFlat" id="StyleBoxFlat_onpxr"] @@ -96,7 +96,6 @@ script = ExtResource("1_anot2") [node name="AudioStreamMicrophone" type="AudioStreamPlayer" parent="."] stream = SubResource("AudioStreamMicrophone_itmlg") -autoplay = true bus = &"MicrophoneBus" [node name="LabelTitle" type="Label" parent="."] @@ -295,6 +294,7 @@ text = "Frame duration" layout_mode = 2 item_count = 6 popup/item_0/text = "2.5ms" +popup/item_0/id = 0 popup/item_1/text = "5ms" popup/item_1/id = 1 popup/item_2/text = "10ms" @@ -382,6 +382,7 @@ layout_mode = 2 allow_reselect = true item_count = 5 popup/item_0/text = "8KHz" +popup/item_0/id = 0 popup/item_1/text = "12KHz" popup/item_1/id = 1 popup/item_2/text = "16KHz" @@ -469,6 +470,7 @@ layout_mode = 2 selected = 0 item_count = 4 popup/item_0/text = "(actions)" +popup/item_0/id = 0 popup/item_1/text = "Save" popup/item_1/id = 1 popup/item_2/text = "Load" @@ -655,6 +657,7 @@ layout_mode = 2 size_flags_horizontal = 3 item_count = 4 popup/item_0/text = "doesliverpool" +popup/item_0/id = 0 popup/item_1/text = "hivemq" popup/item_1/id = 1 popup/item_2/text = "m.org" diff --git a/src/audio_effect_opus_chunked.cpp b/src/audio_effect_opus_chunked.cpp index bd504c2..e0a5ae3 100644 --- a/src/audio_effect_opus_chunked.cpp +++ b/src/audio_effect_opus_chunked.cpp @@ -78,8 +78,60 @@ void AudioEffectOpusChunked::_bind_methods() { ClassDB::bind_method(D_METHOD("resetencoder"), &AudioEffectOpusChunked::resetencoder); ClassDB::bind_method(D_METHOD("chunk_to_opus_packet", "prefixbytes", "audiosamples", "denoise"), &AudioEffectOpusChunked::chunk_to_opus_packet); ClassDB::bind_method(D_METHOD("chunk_resample", "audiosamples", "denoise", "backresample"), &AudioEffectOpusChunked::chunk_resample); + + ClassDB::bind_method(D_METHOD("Drunmicthing"), &AudioEffectOpusChunked::Drunmicthing); +} + +// What have I done: +// in v4.3.stable +// change the ngodot/flake.nix +// substituteInPlace servers/register_server_types.cpp \ +// --replace-fail 'GDREGISTER_CLASS(AudioStreamMicrophone);' 'GDREGISTER_CLASS(AudioStreamMicrophone);GDREGISTER_CLASS(AudioStreamPlaybackMicrophone);' +// +// godot4 --dump-extension-api +// cp extension_api.json ../two-voip-godot-4-DEV/godot-cpp/gdextension/ +// in twovoip run scons +// cp addons/twovoip/libs/libtwovoip.linux.template_debug.x86_64.so example/addons/twovoip_lipsync/libs/ +// +// To compile godot from repositories/godot source code +// nix develop nixpkgs#godot_4 +// scons -j16 dbus=true debug_symbols=false fontconfig=true platform=linuxbsd precision=single production=true pulseaudio=true speechd=true target=editor touch=true udev=true wayland=true x11=true prefix=/home/julian/repositories/godot/outputs/out +// runPhase installPhase +// runphase fixupPhase +// outputs/out/bin/godot4 + + + +void AudioEffectOpusChunked::Drunmicthing() { + if (!DBrunmicthing) { + Daudiostreammicrophoneplayback = Daudiostreammicrophone.instantiate_playback(); + godot::UtilityFunctions::prints("Bingo!"); + godot::UtilityFunctions::prints(Daudiostreammicrophoneplayback); + Daudiostreammicrophoneplayback->begin_resample(); + //Daudiostreammicrophoneplayback->Dingo(); + godot::UtilityFunctions::prints("Bongo!"); + DBrunmicthing = true; + } + + godot::UtilityFunctions::prints(Daudiostreammicrophoneplayback->_is_playing()); + std::vector buff; + buff.resize(882); + int mixed = Daudiostreammicrophoneplayback->_mix_resampled(&buff.front(), 882); + godot::UtilityFunctions::prints(mixed); + + +/* virtual void _start(double p_from_pos); + virtual void _stop(); + virtual bool _is_playing() const; + virtual int32_t _get_loop_count() const; + virtual double _get_playback_position() const; + virtual void _seek(double p_position); + virtual int32_t _mix(AudioFrame *p_buffer, double p_rate_scale, int32_t p_frames); + virtual void _tag_used_streams(); +*/ } + const int MAXPREFIXBYTES = 100; AudioEffectOpusChunked::AudioEffectOpusChunked() { diff --git a/src/audio_effect_opus_chunked.h b/src/audio_effect_opus_chunked.h index cb881b2..e309156 100644 --- a/src/audio_effect_opus_chunked.h +++ b/src/audio_effect_opus_chunked.h @@ -39,6 +39,11 @@ #include #include #include +#include +#include +#include +#include +//#include #include "opus.h" #include "speex_resampler/speex_resampler.h" @@ -58,6 +63,9 @@ namespace godot { + + + class AudioEffectOpusChunked; @@ -140,6 +148,12 @@ class AudioEffectOpusChunked : public AudioEffect { void process(const AudioFrame *p_src_frames, AudioFrame *p_dst_frames, int p_frame_count); + + bool DBrunmicthing = false; + AudioStreamMicrophone Daudiostreammicrophone; + Ref Daudiostreammicrophoneplayback; + + protected: static void _bind_methods(); @@ -154,6 +168,8 @@ class AudioEffectOpusChunked : public AudioEffect { void deleteencoder(); void resetencoder(); + void Drunmicthing(); + bool chunk_available(); void drop_chunk(); bool undrop_chunk();