Merge pull request #775 from vcync/next

Merge next into main
vcync · Nov 1, 2022 · 4ceacea · 4ceacea
2 parents 7923408 + b1ea89f
commit 4ceacea
Show file tree

Hide file tree

Showing 25 changed files with 613 additions and 317 deletions.
diff --git a/package.json b/package.json
@@ -36,7 +36,7 @@
     "fuse.js": "^6.2.1",
     "golden-layout": "^1.5.9",
     "grandiose": "github:vcync/grandiose#feat/workerCompatibility",
-    "interactive-shader-format": "github:vcync/interactive-shader-format-js#fixedIsfParserAndImageBitmap",
+    "interactive-shader-format": "github:vcync/interactive-shader-format-js#78b62b6f4c787b870852df4c8e7b1131e331d8a6",
     "lfo-for-modv": "0.0.1",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",

diff --git a/public/index.html b/public/index.html
@@ -11,7 +11,7 @@
 </head>
 
 <body>
-  <div class="lds-ripple"><div></div><div></div></div>
+  <div class="loading">loading</div>
   <noscript>
     <strong>We're sorry but modv-3.0 doesn't work properly without JavaScript enabled. Please enable it to
       continue.</strong>

diff --git a/public/output-window.html b/public/output-window.html
@@ -1,9 +1,26 @@
 <!-- prettier-ignore -->
 <!DOCTYPE html>
 <html lang="en">
-  <head>
-    <meta charset="utf-8">
-  </head>
 
-  <body style="margin: 0px; display: flex; justify-content: center; align-items: center;"></body>
+<head>
+  <meta charset="utf-8">
+  <style>
+    html,
+    body {
+      height: 100%;
+    }
+
+    body {
+      overflow: hidden;
+      background-color: #000;
+      margin: 0px;
+      display: flex;
+      justify-content: center;
+      align-items: center;
+    }
+  </style>
+</head>
+
+<body></body>
+
 </html>
diff --git a/src/App.vue b/src/App.vue
@@ -403,39 +403,11 @@ body {
 }
 
 /* From https://loading.io/css */
-.lds-ripple {
+.loading {
   position: absolute;
-  top: 50%;
   left: 50%;
-  transform: translate(-50%, -50%) scale(2.5);
-  width: 80px;
-  height: 80px;
-}
-.lds-ripple div {
-  position: absolute;
-  border: 4px solid rgb(21, 21, 21);
-  opacity: 1;
-  border-radius: 100%;
-  animation: lds-ripple 1s cubic-bezier(0, 0.2, 0.8, 1) infinite;
-}
-.lds-ripple div:nth-child(2) {
-  animation-delay: -0.5s;
-}
-@keyframes lds-ripple {
-  0% {
-    top: 36px;
-    left: 36px;
-    width: 0;
-    height: 0;
-    opacity: 1;
-  }
-  100% {
-    top: 0px;
-    left: 0px;
-    width: 72px;
-    height: 72px;
-    opacity: 0;
-  }
+  top: 50%;
+  transform: translate(-50%, -50%);
 }
 </style>
 

diff --git a/src/application/constants.js b/src/application/constants.js
@@ -15,3 +15,7 @@ export default {
     return 512;
   }
 };
+
+export const GROUP_DISABLED = 0;
+export const GROUP_ENABLED = 1;
+export const GROUP_DRAW_TO_OUTPUT = 2;
diff --git a/src/application/index.js b/src/application/index.js
@@ -2,7 +2,8 @@ import Worker from "worker-loader!./worker/index.worker.js";
 import {
   setupMedia,
   enumerateDevices,
-  getByteFrequencyData
+  getByteFrequencyData,
+  getByteTimeDomainData
 } from "./setup-media";
 import setupBeatDetektor from "./setup-beat-detektor";
 import setupMidi from "./setup-midi";
@@ -15,6 +16,7 @@ import PromiseWorker from "promise-worker-transferable";
 import Vue from "vue";
 import { ipcRenderer } from "electron";
 import { app } from "@electron/remote";
+import { GROUP_ENABLED } from "./constants";
 
 let imageBitmap;
 const imageBitmapQueue = [];
@@ -132,7 +134,7 @@ export default class ModV {
     };
 
     // Make the default group
-    this.store.dispatch("groups/createGroup", { enabled: true });
+    this.store.dispatch("groups/createGroup", { enabled: GROUP_ENABLED });
 
     window.addEventListener("beforeunload", () => true);
   }
@@ -141,7 +143,7 @@ export default class ModV {
     this.windowHandler();
 
     try {
-      await this.setupMedia({});
+      await this.setupMedia({ useDefaultDevices: true });
     } catch (e) {
       console.error(e);
     }
@@ -259,13 +261,17 @@ export default class ModV {
     } = this.store.state;
 
     const features = this.meyda.get(featuresToGet);
-    this.updateBeatDetektor(delta, features);
-    features.byteFrequencyData = Array.from(getByteFrequencyData() || []);
-    this.$worker.postMessage({ type: "meyda", payload: features });
 
-    for (let i = 0; i < featuresToGet.length; i += 1) {
-      const feature = featuresToGet[i];
-      this.features[feature] = features[feature];
+    if (features) {
+      this.updateBeatDetektor(delta, features);
+      features.byteFrequencyData = Array.from(getByteFrequencyData() || []);
+      features.byteTimeDomainData = Array.from(getByteTimeDomainData() || []);
+      this.$worker.postMessage({ type: "meyda", payload: features });
+
+      for (let i = 0; i < featuresToGet.length; i += 1) {
+        const feature = featuresToGet[i];
+        this.features[feature] = features[feature];
+      }
     }
   }
 

diff --git a/src/application/renderers/2d.js b/src/application/renderers/2d.js
@@ -65,9 +65,35 @@ function render({
   context.drawImage(twoDCanvas, 0, 0, canvas.width, canvas.height);
 }
 
+/**
+ * Called each frame to update the Module
+ */
+function updateModule({
+  moduleDefinition,
+  props,
+  data,
+  canvas,
+  context,
+  delta
+}) {
+  const { data: dataUpdated } = moduleDefinition.update({
+    props,
+    data,
+    canvas,
+    context,
+    delta
+  });
+
+  return dataUpdated ?? data;
+}
+
+function resizeModule({ moduleDefinition, canvas, data, props }) {
+  return moduleDefinition.resize({ canvas, data, props });
+}
+
 function resize({ width, height }) {
   twoDCanvas.width = width;
   twoDCanvas.height = height;
 }
 
-export default { render, resize };
+export default { render, resize, updateModule, resizeModule };
diff --git a/src/application/renderers/isf.js b/src/application/renderers/isf.js
@@ -1,12 +1,12 @@
 import store from "../worker/store";
 
-import { interactiveShaderFormat } from "interactive-shader-format/dist/build-worker";
-
-const {
-  Renderer: ISFRenderer,
-  Parser: ISFParser,
-  Upgrader: ISFUpgrader
-} = interactiveShaderFormat;
+import {
+  Renderer as ISFRenderer,
+  Parser as ISFParser,
+  Upgrader as ISFUpgrader
+} from "interactive-shader-format/src/main.js";
+import { getFeatures } from "../worker/audio-features";
+import constants from "../constants";
 
 const isfCanvas = new OffscreenCanvas(300, 300);
 const isfContext = isfCanvas.getContext("webgl2", {
@@ -33,6 +33,17 @@ function render({ module, canvas, context, pipeline, props }) {
   const renderer = renderers[module.meta.name];
   const moduleInputs = inputs[module.meta.name];
 
+  // Only update the audio data if the module has audio inputs to improve performance
+  // for modules that don't use any audio inputs at all
+  if (renderer.hasAudio) {
+    const features = getFeatures();
+    const byteFrequencyData = features.byteFrequencyData;
+    const byteTimeDomainData = features.byteTimeDomainData;
+
+    renderer.audio.setFrequencyValues(byteFrequencyData, byteFrequencyData);
+    renderer.audio.setTimeDomainValues(byteTimeDomainData, byteTimeDomainData);
+  }
+
   if (moduleInputs) {
     for (let i = 0, len = moduleInputs.length; i < len; i++) {
       const input = moduleInputs[i];
@@ -60,9 +71,28 @@ function render({ module, canvas, context, pipeline, props }) {
   context.drawImage(isfCanvas, 0, 0, canvas.width, canvas.height);
 }
 
-async function setupModule(module) {
-  let fragmentShader = module.fragmentShader;
-  let vertexShader = module.vertexShader;
+/**
+ * Called each frame to update the Module
+ */
+function updateModule({ module, props, data, canvas, context, delta }) {
+  const { data: dataUpdated } = module.update({
+    props,
+    data,
+    canvas,
+    context,
+    delta
+  });
+
+  return dataUpdated ?? data;
+}
+
+function resizeModule({ moduleDefinition, canvas, data, props }) {
+  return moduleDefinition.resize({ canvas, data, props });
+}
+
+async function setupModule(moduleDefinition) {
+  let fragmentShader = moduleDefinition.fragmentShader;
+  let vertexShader = moduleDefinition.vertexShader;
 
   const parser = new ISFParser();
   parser.parse(fragmentShader, vertexShader);
@@ -77,24 +107,28 @@ async function setupModule(module) {
     }
   }
 
-  module.meta.isfVersion = parser.isfVersion;
-  module.meta.author = parser.metadata.CREDIT;
-  module.meta.description = parser.metadata.DESCRIPTION;
-  module.meta.version = parser.metadata.VSN;
+  moduleDefinition.meta.isfVersion = parser.isfVersion;
+  moduleDefinition.meta.author = parser.metadata.CREDIT;
+  moduleDefinition.meta.description = parser.metadata.DESCRIPTION;
+  moduleDefinition.meta.version = parser.metadata.VSN;
 
-  const renderer = new ISFRenderer(isfContext);
+  const renderer = new ISFRenderer(isfContext, {
+    useWebAudio: false,
+    fftSize: constants.AUDIO_BUFFER_SIZE,
+    hasAudio: parser.hasAudio
+  });
   renderer.loadSource(fragmentShader, vertexShader);
 
   if (!renderer.valid) {
     throw renderer.errorWithCorrectedLines;
   }
 
   function addProp(name, prop) {
-    if (!module.props) {
-      module.props = {};
+    if (!moduleDefinition.props) {
+      moduleDefinition.props = {};
     }
 
-    module.props[name] = prop;
+    moduleDefinition.props[name] = prop;
   }
 
   const moduleInputs = parser.inputs;
@@ -155,7 +189,7 @@ async function setupModule(module) {
         break;
 
       case "image":
-        module.meta.previewWithOutput = true;
+        moduleDefinition.meta.previewWithOutput = true;
 
         addProp(input.NAME, {
           type: "texture",
@@ -166,15 +200,17 @@ async function setupModule(module) {
     }
   }
 
-  renderers[module.meta.name] = renderer;
-  inputs[module.meta.name] = moduleInputs;
-  module.draw = render;
+  renderers[moduleDefinition.meta.name] = renderer;
+  inputs[moduleDefinition.meta.name] = moduleInputs;
+  moduleDefinition.draw = render;
 
-  return module;
+  return moduleDefinition;
 }
 
 export default {
   setupModule,
   render,
+  updateModule,
+  resizeModule,
   resize
 };