From 71ffad40d2eb2709d0a30168b558d1a2500dd2c9 Mon Sep 17 00:00:00 2001 From: PolyMeilex Date: Thu, 1 Jun 2023 02:22:27 +0200 Subject: [PATCH] POC: Neothesia Web --- Cargo.lock | 51 +++- Cargo.toml | 1 + midi-file/Cargo.toml | 2 +- midi-file/src/midi.rs | 6 +- neothesia-core/src/utils/resources.rs | 6 + neothesia-pipelines/src/waterfall/mod.rs | 14 +- neothesia-pipelines/src/waterfall/shader.wgsl | 3 + neothesia-web/.gitignore | 1 + neothesia-web/Cargo.toml | 25 ++ neothesia-web/README.md | 4 + neothesia-web/index.html | 35 +++ neothesia-web/src/main.rs | 239 ++++++++++++++++++ wgpu-jumpstart/src/gpu.rs | 4 + 13 files changed, 383 insertions(+), 8 deletions(-) create mode 100644 neothesia-web/.gitignore create mode 100644 neothesia-web/Cargo.toml create mode 100644 neothesia-web/README.md create mode 100644 neothesia-web/index.html create mode 100644 neothesia-web/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index ea751951..46121d81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -512,6 +512,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "console_log" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8aed40e4edbf4d3b4431ab260b63fdc40f5780a4766824329ea0f1eefe3c0f" +dependencies = [ + "log", + "web-sys", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -1868,9 +1888,6 @@ name = "midly" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "207d755f4cb882d20c4da58d707ca9130a0c9bc5061f657a4f299b8e36362b7a" -dependencies = [ - "rayon", -] [[package]] name = "minimal-lexical" @@ -2052,6 +2069,28 @@ dependencies = [ "wgpu-jumpstart", ] +[[package]] +name = "neothesia-web" +version = "0.1.0" +dependencies = [ + "console_error_panic_hook", + "console_log", + "env_logger", + "getrandom", + "instant", + "log", + "midi-file", + "neothesia-core", + "piano-math", + "pollster", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu", + "wgpu-jumpstart", + "winit", +] + [[package]] name = "nix" version = "0.24.3" @@ -2604,6 +2643,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "pollster" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2" + [[package]] name = "ppv-lite86" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index 3ffb51a4..0f9e6111 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "neothesia-cli", "neothesia-core", "neothesia-pipelines", + "neothesia-web", "midi-file", "midi-io", ] diff --git a/midi-file/Cargo.toml b/midi-file/Cargo.toml index b6eaae0d..c51b5ae8 100644 --- a/midi-file/Cargo.toml +++ b/midi-file/Cargo.toml @@ -5,4 +5,4 @@ authors = ["Poly "] edition = "2021" [dependencies] -midly = "0.5" +midly = { version = "0.5", default-features = false, features = ["std"] } diff --git a/midi-file/src/midi.rs b/midi-file/src/midi.rs index d031007d..a8438f23 100644 --- a/midi-file/src/midi.rs +++ b/midi-file/src/midi.rs @@ -16,7 +16,11 @@ impl Midi { Err(_) => return Err(String::from("Could Not Open File")), }; - let smf = match Smf::parse(&data) { + Self::new_from_bytes(&data) + } + + pub fn new_from_bytes(data: &[u8]) -> Result { + let smf = match Smf::parse(data) { Ok(smf) => smf, Err(_) => return Err(String::from("Midi Parsing Error (midly lib)")), }; diff --git a/neothesia-core/src/utils/resources.rs b/neothesia-core/src/utils/resources.rs index e4170bf4..d2060f4f 100644 --- a/neothesia-core/src/utils/resources.rs +++ b/neothesia-core/src/utils/resources.rs @@ -48,6 +48,9 @@ pub fn default_sf2() -> Option { #[cfg(target_os = "macos")] return bundled_resource_path("default", "sf2").map(PathBuf::from); + + #[cfg(target_family = "wasm")] + return None; } pub fn settings_ron() -> Option { @@ -59,6 +62,9 @@ pub fn settings_ron() -> Option { #[cfg(target_os = "macos")] return bundled_resource_path("settings", "ron").map(PathBuf::from); + + #[cfg(target_family = "wasm")] + return None; } #[cfg(target_os = "macos")] diff --git a/neothesia-pipelines/src/waterfall/mod.rs b/neothesia-pipelines/src/waterfall/mod.rs index d24288dd..b7458828 100644 --- a/neothesia-pipelines/src/waterfall/mod.rs +++ b/neothesia-pipelines/src/waterfall/mod.rs @@ -26,7 +26,7 @@ impl<'a> WaterfallPipeline { let shader = gpu .device .create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("RectanglePipeline::shader"), + label: Some("waterfall::shader"), source: wgpu::ShaderSource::Wgsl(std::borrow::Cow::Borrowed(include_str!( "./shader.wgsl" ))), @@ -41,7 +41,7 @@ impl<'a> WaterfallPipeline { let render_pipeline_layout = &gpu.device .create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: None, + label: Some("waterfall::pipeline"), bind_group_layouts: &[ &transform_uniform.bind_group_layout, &time_uniform.bind_group_layout, @@ -106,10 +106,18 @@ impl<'a> WaterfallPipeline { #[derive(Clone, Copy, Pod, Zeroable)] struct TimeUniform { time: f32, + _pad1: f32, + _pad2: f32, + _pad3: f32, } impl Default for TimeUniform { fn default() -> Self { - Self { time: 0.0 } + Self { + time: 0.0, + _pad1: 0.0, + _pad2: 0.0, + _pad3: 0.0, + } } } diff --git a/neothesia-pipelines/src/waterfall/shader.wgsl b/neothesia-pipelines/src/waterfall/shader.wgsl index 349525fb..e0f3c93d 100644 --- a/neothesia-pipelines/src/waterfall/shader.wgsl +++ b/neothesia-pipelines/src/waterfall/shader.wgsl @@ -6,6 +6,9 @@ struct ViewUniform { struct TimeUniform { time: f32, + _pad1: f32, + _pad2: f32, + _pad3: f32, } @group(0) @binding(0) diff --git a/neothesia-web/.gitignore b/neothesia-web/.gitignore new file mode 100644 index 00000000..53c37a16 --- /dev/null +++ b/neothesia-web/.gitignore @@ -0,0 +1 @@ +dist \ No newline at end of file diff --git a/neothesia-web/Cargo.toml b/neothesia-web/Cargo.toml new file mode 100644 index 00000000..df72f872 --- /dev/null +++ b/neothesia-web/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "neothesia-web" +version = "0.1.0" +edition = "2021" + +[dependencies] +neothesia-core = { workspace = true } +wgpu = { workspace = true, features = ["webgl"] } +wgpu-jumpstart = { workspace = true } + +winit = { version = "0.28.2" } +env_logger = { workspace = true } +pollster = "0.3.0" +console_log = "1.0.0" +web-sys = "0.3.61" +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4.34" +console_error_panic_hook = "0.1.7" + +midi-file = { workspace = true } +piano-math = { workspace = true } +log = { workspace = true } +instant = "0.1.12" + +getrandom = { version = "0.2", features = ["js"] } diff --git a/neothesia-web/README.md b/neothesia-web/README.md new file mode 100644 index 00000000..cbe0e866 --- /dev/null +++ b/neothesia-web/README.md @@ -0,0 +1,4 @@ +# Run +``` +trunk serve --open +``` \ No newline at end of file diff --git a/neothesia-web/index.html b/neothesia-web/index.html new file mode 100644 index 00000000..c49576cd --- /dev/null +++ b/neothesia-web/index.html @@ -0,0 +1,35 @@ + + + + + + + Trunk | Vanilla | web-sys + + + + + + + + + + + \ No newline at end of file diff --git a/neothesia-web/src/main.rs b/neothesia-web/src/main.rs new file mode 100644 index 00000000..2c85f1ed --- /dev/null +++ b/neothesia-web/src/main.rs @@ -0,0 +1,239 @@ +use std::time::Duration; + +use winit::{ + event::{Event, WindowEvent}, + event_loop::{ControlFlow, EventLoop}, + window::Window, +}; + +use wgpu_jumpstart::Gpu; + +async fn run(event_loop: EventLoop<()>, window: Window) { + let midi = midi_file::Midi::new_from_bytes(include_bytes!("../../test.mid")).unwrap(); + let mut playback = midi_file::PlaybackState::new(Duration::from_secs(3), &midi.merged_track); + + let size = window.inner_size(); + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::default()); + + let surface = unsafe { instance.create_surface(&window) }.unwrap(); + let mut gpu = Gpu::new(&instance, Some(&surface)).await.unwrap(); + + let width = size.width; + let height = size.height; + + let mut transform_uniform = wgpu_jumpstart::TransformUniform::default(); + transform_uniform.update(width as f32, height as f32, 1.0); + + let transform_uniform = wgpu_jumpstart::Uniform::new( + &gpu.device, + transform_uniform, + wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ); + + let keyboard_layout = get_layout(width as f32, height as f32); + + let mut keyboard = neothesia_core::render::KeyboardRenderer::new( + &gpu, + &transform_uniform, + keyboard_layout.clone(), + ); + + keyboard.position_on_bottom_of_parent(height as f32); + + let neothesia_config = neothesia_core::config::Config::new(); + let mut waterfall = neothesia_core::render::WaterfallRenderer::new( + &gpu, + &midi, + &neothesia_config, + &transform_uniform, + keyboard_layout, + ); + + let mut text = neothesia_core::render::TextRenderer::new(&gpu); + + // keyboard.update(&gpu.queue, text.glyph_brush()); + + // + + // Load the shaders from disk + + let capabilities = surface.get_capabilities(&gpu.adapter); + let swapchain_format = capabilities.formats[0]; + + let mut config = wgpu::SurfaceConfiguration { + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + format: swapchain_format, + view_formats: vec![swapchain_format], + width: size.width, + height: size.height, + present_mode: wgpu::PresentMode::Fifo, + alpha_mode: capabilities.alpha_modes[0], + }; + + surface.configure(&gpu.device, &config); + + let mut last_time = instant::Instant::now(); + + event_loop.run(move |event, _, control_flow| { + // Have the closure take ownership of the resources. + // `event_loop.run` never returns, therefore we must do this to ensure + // the resources are properly cleaned up. + let _ = (&instance, &gpu.adapter); + + match event { + Event::WindowEvent { + event: WindowEvent::Resized(size), + .. + } => { + // Reconfigure the surface with the new size + config.width = size.width; + config.height = size.height; + surface.configure(&gpu.device, &config); + // On macos the window needs to be redrawn manually after resizing + window.request_redraw(); + } + Event::RedrawRequested(_) => { + let frame = surface + .get_current_texture() + .expect("Failed to acquire next swap chain texture"); + let view = frame + .texture + .create_view(&wgpu::TextureViewDescriptor::default()); + let mut encoder = gpu + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None }); + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: None, + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: true, + }, + })], + depth_stencil_attachment: None, + }); + + waterfall.render(&transform_uniform, &mut rpass); + keyboard.render(&transform_uniform, &mut rpass); + } + + text.render((width as f32, height as f32), &mut gpu, &view); + + gpu.queue.submit(Some(encoder.finish())); + frame.present(); + } + Event::WindowEvent { + event: WindowEvent::CloseRequested, + .. + } => *control_flow = ControlFlow::Exit, + Event::MainEventsCleared => { + let delta = last_time.elapsed(); + last_time = instant::Instant::now(); + + let events = playback.update(&midi.merged_track, delta); + file_midi_events(&mut keyboard, &neothesia_config, &events); + + waterfall.update(&gpu.queue, time_without_lead_in(&playback)); + + keyboard.update(&gpu.queue, text.glyph_brush()); + + window.request_redraw(); + } + _ => {} + } + }); +} + +fn main() { + let event_loop = EventLoop::new(); + let window = winit::window::Window::new(&event_loop).unwrap(); + #[cfg(not(target_arch = "wasm32"))] + { + env_logger::init(); + // Temporarily avoid srgb formats for the swapchain on the web + pollster::block_on(run(event_loop, window)); + } + #[cfg(target_arch = "wasm32")] + { + std::panic::set_hook(Box::new(console_error_panic_hook::hook)); + console_log::init().expect("could not initialize logger"); + use winit::platform::web::WindowExtWebSys; + + on_resize(&window); + + // On wasm, append the canvas to the document body + web_sys::window() + .and_then(|win| win.document()) + .and_then(|doc| doc.body()) + .and_then(|body| { + body.append_child(&web_sys::Element::from(window.canvas())) + .ok() + }) + .expect("couldn't append canvas to document body"); + wasm_bindgen_futures::spawn_local(run(event_loop, window)); + } +} + +#[cfg(target_arch = "wasm32")] +fn on_resize(window: &Window) { + use winit::dpi::{LogicalSize, PhysicalSize}; + + let body = web_sys::window() + .and_then(|win| win.document()) + .and_then(|doc| doc.body()); + + if let Some(body) = body { + let (width, height) = (body.client_width(), body.client_height()); + + let factor = window.scale_factor(); + let logical = LogicalSize { width, height }; + let size: PhysicalSize = logical.to_physical(factor); + + window.set_inner_size(size) + } +} + +fn get_layout(width: f32, height: f32) -> piano_math::KeyboardLayout { + let white_count = piano_math::KeyboardRange::standard_88_keys().white_count(); + let neutral_width = width / white_count as f32; + let neutral_height = height * 0.2; + + piano_math::standard_88_keys(neutral_width, neutral_height) +} + +pub fn file_midi_events( + keyboard: &mut neothesia_core::render::KeyboardRenderer, + config: &neothesia_core::config::Config, + events: &[midi_file::MidiEvent], +) { + use midi_file::midly::MidiMessage; + + for e in events { + let (is_on, key) = match e.message { + MidiMessage::NoteOn { key, .. } => (true, key.as_int()), + MidiMessage::NoteOff { key, .. } => (false, key.as_int()), + _ => continue, + }; + + if keyboard.range().contains(key) && e.channel != 9 { + let id = key as usize - 21; + let key = &mut keyboard.key_states_mut()[id]; + + if is_on { + let color = &config.color_schema[e.track_id % config.color_schema.len()]; + key.pressed_by_file_on(color); + } else { + key.pressed_by_file_off(); + } + + keyboard.queue_reupload(); + } + } +} + +pub fn time_without_lead_in(playback: &midi_file::PlaybackState) -> f32 { + playback.time().as_secs_f32() - playback.leed_in().as_secs_f32() +} diff --git a/wgpu-jumpstart/src/gpu.rs b/wgpu-jumpstart/src/gpu.rs index 0ca52705..a6376fca 100644 --- a/wgpu-jumpstart/src/gpu.rs +++ b/wgpu-jumpstart/src/gpu.rs @@ -51,6 +51,10 @@ impl Gpu { .request_device( &wgpu::DeviceDescriptor { features: wgpu::Features::empty(), + #[cfg(target_family = "wasm")] + limits: wgpu::Limits::downlevel_webgl2_defaults() + .using_resolution(adapter.limits()), + #[cfg(not(target_family = "wasm"))] limits: wgpu::Limits { max_compute_workgroup_storage_size: 0, max_compute_invocations_per_workgroup: 0,