Merge pull request #82 from Absulit/dev

Dev

README.md

@@ -70,9 +70,8 @@ You can have a Vertex + Fragment shaders without a Compute shader, and also a Co
 ```js
 // import the `Points` class
 
-import Points, {
-    RenderPass
-} from '../src/absulit.points.module.js';
+import Points from '../src/absulit.points.module.js';
+import RenderPass from '../src/RenderPass.js';
 
 // reference the canvas in the constructor
 const points = new Points('gl-canvas');
@@ -455,7 +454,54 @@ Common uses:
 > - name - name this property/variable will have inside the shader
 > - size - number of items it will allocate
 > - structName - You can use one of the default structs/types like `f32` , `i32` , `u32` , but if you use a more complex one you have to pair it properly with structSize. If it's a custom `struct` it has to be declared in the shader or it will throw an error.
-> - structSize - if the `struct` you reference in `structName` has 4 properties then you have to add `4` . If it's only a f32 then here you should place `1` .
+> - structSize - if the `struct` you reference in `structName` has 4 properties then you have to add `4` . If it's only a f32 then here you should place `1` . If the `struct` has vectors in it, the size of the vector counts:
+
+```rust
+// structSize: 1
+f32
+u32
+i32
+
+// structSize: 2
+vec2<f32>
+vec2<i32>
+vec2<u32>
+vec2f
+vec2i
+vec2u
+
+// structSize: 3
+vec3<f32>
+vec3<i32>
+vec3<u32>
+vec3f
+vec3i
+vec3u
+
+// structSize: 4
+vec4<f32>
+vec4<i32>
+vec4<u32>
+vec4f
+vec4i
+vec4u
+
+// CUSTOM STRUCTS
+
+// structSize: 3
+struct Variables {
+    value1: vec2f,
+    value2: f32,
+}
+
+// structSize: 5
+struct Variables2 {
+    value1: vec3f,
+    value2: i32,
+    value3: u32,
+}
+
+```
 
 ---
 
@@ -498,6 +544,13 @@ let b = value_noise_data[0];
 variables.isCreated = 1;
 ```
 
+You can also add a default type instead of a custom struct in `structName`:
+
+```js
+points.addStorage('myVar', 1, 'f32', 1); // size is 1
+points.addStorage('myVar2', 1, 'vec2f', 2); // size is 2
+```
+
 ## StorageMap - addStorageMap
 
 Creates a Storage in the same way as a `addStorage` does, except it can be set with data from the start of the application.
@@ -592,6 +645,70 @@ async function init() {
 let rgbaWebcam = textureSampleBaseClampToEdge(webcam, feedbackSampler, fract(uv));
 ```
 
+## Audio - addAudio
+
+You can load audio and use its data for visualization.
+
+```js
+// index.js
+let audio = points.addAudio('myAudio', './../../audio/cognitive_dissonance.mp3', volume, loop, false);
+```
+
+
+With the `myAudio` name, a `Sound` type named `myAudio` is created. In the future it will have more information but now it only has the `data` property. `data` is an `array<f32, 2048>`, but it's not completely filled with data, it's only filled up to `params.myAudioLength`, (`myAudio` used as prefix for each different audio) and then each of these values has a max of 256, so if you want something like a percentage, you have to divide the value at a certain index between 256
+```rust
+let audioX = audio.data[ u32(uvr.x * params.audioLength)] / 256;
+```
+
+
+---
+
+> **Note:** The `points.addAudio` method returns a `new Audio` reference, you are responsible to start and stop the audio from the JavaScript side, if you require to start and stop a sound by creating a call from the shaders, please check the `Events - addEventListener` section
+
+---
+
+## Events - addEventListener
+
+An event is an asynchronous message that can be send from the WGSL shaders to the JavaScript code. It works in a very similar way as to listen for a click in JavaScript or a screen resize, you call a `addEventListener`, but the parameters and its use change a bit. In the code below, the first parameter is the name of the event you , the event is fired by **you**, so this name currently has no predefined names like 'click', or 'mouse over', you have to define them and dispatch them.
+
+The second parameter is the data (if any) that you will send from the WGSL shaders, this data is returnes as an array, so you have to acces each item by its index.
+
+The last parameter is the amount of items in this array you are expecting. All items are `f32`.
+
+```js
+// as in examples\events1\index.js
+points.addEventListener(
+    'right_blink', // name of the event (and name of a storage)
+    data => { // data returned after the event and callback
+        console.log('---- Right Circle');
+    },
+    4 // size of the data to return
+);
+```
+
+To fire an event you first need to declare the listener. The name used in the listener is also used as storage buffer that you can manipulate to dispatch the event.
+
+The event has the following structure:
+
+```rust
+// as in src\core\defaultStructs.js
+struct Event {
+    updated: u32,
+    data: array<f32>
+}
+```
+
+To actully fire an event you have to do as follows:
+
+```rust
+right_blink.data[0] = 2; // some data
+right_blink.data[1] = 2; // some data
+right_blink.updated = 1; // update this property to something diff than `0`
+```
+
+By just simply changing the value of `updated` to a non zero, the library knows this event has been updated, and will dispatch the event immediately in JavaScript, so the `console.log` will print the text in the JavaScript Console. `updated` will be set as zero in the next frame, and if not updated the library then knows it doesn't have to do anything.
+
+
 ## Layers - addLayers
 
 A layer is basically a Storage but pre-made with the exact same dimension of the canvas, this for potentially create multi-layered effects that require a type of temporary storage and swap values between them. All items are `vec4<f32>`

audio/cognitive_dissonance

@@ -0,0 +1,3 @@
+"Cognitive Dissonance" Kevin MacLeod (incompetech.com)
+Licensed under Creative Commons: By Attribution 4.0 License
+http://creativecommons.org/licenses/by/4.0/

examples/audio1/compute.js

@@ -0,0 +1,13 @@
+const compute = /*wgsl*/`
+
+@compute @workgroup_size(8,8,1)
+fn main(
+    @builtin(global_invocation_id) GlobalId: vec3<u32>,
+    @builtin(workgroup_id) WorkGroupID: vec3<u32>,
+    @builtin(local_invocation_id) LocalInvocationID: vec3<u32>
+) {
+    let time = params.time;
+}
+`;
+
+export default compute;

examples/audio1/frag.js

@@ -0,0 +1,43 @@
+import { fnusin } from '../../src/core/animation.js';
+import { snoise } from './../../src/core/noise2d.js';
+import { sdfCircle } from './../../src/core/sdf.js';
+import { WHITE, RED, layer } from './../../src/core/color.js';
+import { audioAverage, audioAverageSegments } from '../../src/core/audio.js';
+
+const frag = /*wgsl*/`
+
+${fnusin}
+${snoise}
+${sdfCircle}
+${layer}
+${audioAverage}
+${audioAverageSegments}
+${WHITE}
+${RED}
+
+@fragment
+fn main(
+    @location(0) color: vec4<f32>,
+    @location(1) uv: vec2<f32>,
+    @location(2) ratio: vec2<f32>,  // relation between params.screenWidth and params.screenHeight
+    @location(3) uvr: vec2<f32>,    // uv with aspect ratio corrected
+    @location(4) mouse: vec2<f32>,
+    @builtin(position) position: vec4<f32>
+) -> @location(0) vec4<f32> {
+
+    let audioX = audio.data[ u32(uvr.x * params.audioLength)] / 256;
+
+    if(params.mouseClick == 1.){
+        click_event.updated = 1;
+    }
+
+
+    var c = vec4f();
+    c.r = audioX;
+    c.a = 1.;
+
+    return c;
+}
+`;
+
+export default frag;

examples/audio1/index.js

@@ -0,0 +1,40 @@
+import vert from './vert.js';
+import compute from './compute.js';
+import frag from './frag.js';
+import Points from './../../src/absulit.points.module.js';
+
+let audio = null;
+
+const base = {
+    vert,
+    compute,
+    frag,
+    /**
+     *
+     * @param {Points} points
+     */
+    init: async points => {
+        let volume = 1;
+        let loop = true;
+        // audio = points.addAudio('audio', './../../audio/generative_audio_test.ogg', volume, loop);
+        audio = points.addAudio('audio', './../../audio/cognitive_dissonance.mp3', volume, loop, false);
+        points.addEventListener('click_event', data => {
+            audio.play();
+        }, 2);
+        // points.addAudio('audio', 'https://mdn.github.io/voice-change-o-matic/audio/concert-crowd.ogg', volume, loop);
+        points.addStorage('result', 10, 'f32', 1);
+    },
+    /**
+     *
+     * @param {Points} points
+     */
+    update: points => {
+    },
+
+    remove: _ => {
+        audio.pause();
+        audio.remove();
+    }
+}
+
+export default base;

examples/audio1/vert.js

@@ -0,0 +1,15 @@
+const vert = /*wgsl*/`
+
+@vertex
+fn main(
+    @location(0) position: vec4<f32>,
+    @location(1) color: vec4<f32>,
+    @location(2) uv: vec2<f32>,
+    @builtin(vertex_index) vertexIndex: u32
+) -> Fragment {
+
+    return defaultVertexBody(position, color, uv);
+}
+`;
+
+export default vert;

examples/audio2/compute.js

@@ -0,0 +1,13 @@
+const compute = /*wgsl*/`
+
+@compute @workgroup_size(8,8,1)
+fn main(
+    @builtin(global_invocation_id) GlobalId: vec3<u32>,
+    @builtin(workgroup_id) WorkGroupID: vec3<u32>,
+    @builtin(local_invocation_id) LocalInvocationID: vec3<u32>
+) {
+    let time = params.time;
+}
+`;
+
+export default compute;

examples/audio2/frag.js

@@ -0,0 +1,71 @@
+import { fnusin } from '../../src/core/animation.js';
+import { snoise } from '../../src/core/noise2d.js';
+import { sdfCircle } from '../../src/core/sdf.js';
+import { WHITE, RED, GREEN, YELLOW, layer } from '../../src/core/color.js';
+import { audioAverage, audioAverageSegments } from '../../src/core/audio.js';
+import { texturePosition } from '../../src/core/image.js';
+
+const frag = /*wgsl*/`
+
+${fnusin}
+${snoise}
+${sdfCircle}
+${layer}
+${audioAverage}
+${audioAverageSegments}
+${WHITE}
+${RED}
+${GREEN}
+${YELLOW}
+${texturePosition}
+
+@fragment
+fn main(
+    @location(0) color: vec4<f32>,
+    @location(1) uv: vec2<f32>,
+    @location(2) ratio: vec2<f32>,  // relation between params.screenWidth and params.screenHeight
+    @location(3) uvr: vec2<f32>,    // uv with aspect ratio corrected
+    @location(4) mouse: vec2<f32>,
+    @builtin(position) position: vec4<f32>
+) -> @location(0) vec4<f32> {
+
+    if(params.mouseClick == 1.){
+        click_event.updated = 1;
+    }
+
+    // let audioAverage = audioAverage(audio);
+    // let audioAverageSegments = audioAverageSegments(2);
+
+    let n = snoise(uvr / params.sliderA + params.time);
+    let feedbackColor = texturePosition(feedbackTexture, imageSampler, vec2(), uvr * vec2f(1, 1.01), true);
+
+    let segmentNum = 4;
+    let subSegmentLength = i32(params.audioLength) / segmentNum;
+
+    for (var index = 0; index < segmentNum ; index++) {
+        var audioAverage = 0.;
+        for (var index2 = 0; index2 < subSegmentLength; index2++) {
+            let audioIndex = index2 * index;
+
+            let audioValue = audio.data[audioIndex] / 256;
+            audioAverage += audioValue;
+        }
+        result[index] = audioAverage / f32(subSegmentLength);
+    }
+
+
+    let circle1 = sdfCircle(vec2(.5), result[0] * .4, .0, uvr) * WHITE;
+    let circle2 = sdfCircle(vec2(.5), result[1] * .4, .0, uvr) * GREEN;
+    let circle3 = sdfCircle(vec2(.5), result[2] * .4, .0, uvr) * YELLOW;
+    let circle4 = sdfCircle(vec2(.5), result[3] * .4, .0, uvr) * RED;
+
+
+    // return c;
+    // return layer(circle2 * WHITE, c + circle);
+    // return c + circle - circle2;
+
+    return  layer(feedbackColor * .9, layer(circle1, layer(circle2, layer(circle3, circle4))));
+}
+`;
+
+export default frag;