Merge branch 'release/light-adjustment-2023.2.0'

shiguredo · Mar 29, 2023 · b72af66 · b72af66
2 parents f42852f + b0874cb
commit b72af66
Show file tree

Hide file tree

Showing 8 changed files with 100 additions and 33 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -16,6 +16,12 @@
     - npm には公開せず、このリポジトリの内部からのみ参照される想定
     - @sile
 
+## light-adjustment-2023.2.0
+
+- [UPDATE] 複数の映像プロセッサを併用できるようにする
+    - 現状では Chrome / Edge のみで対応
+    - @sile
+
 ## light-adjustment-2023.1.0
 
 **初リリース**

diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
-仮想背景やノイズ抑制といったメディア処理をブラウザで簡単に行えるようにするためのライブラリです。
+仮想背景やノイズ抑制、ライト調整といったメディア処理をブラウザで簡単に行えるようにするためのライブラリです。
 
 ## About Shiguredo's open source software
 
@@ -23,18 +23,21 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use
 
 - [仮想背景 / 背景ぼかし](https://github.com/shiguredo/media-processors/tree/develop/packages/virtual-background)
 - [ノイズ抑制](https://github.com/shiguredo/media-processors/tree/develop/packages/noise-suppression)
+- [ライト調整](https://github.com/shiguredo/media-processors/tree/develop/packages/light-adjustment)
 
 ## npm
 
 - [@shiguredo/virtual\-background \- npm](https://www.npmjs.com/package/@shiguredo/virtual-background)
 - [@shiguredo/noise\-suppression \- npm](https://www.npmjs.com/package/@shiguredo/noise-suppression)
+- [@shiguredo/light\-adjustment \- npm](https://www.npmjs.com/package/@shiguredo/light-adjustment)
 
 ## サンプル
 
 GitHub Pages にサンプルを用意しています。完全にクライアントでのみ動作します。
 
 - [仮想背景 / 背景ぼかし](https://shiguredo.github.io/media-processors/examples/virtual-background.html)
 - [ノイズ抑制](https://shiguredo.github.io/media-processors/examples/noise-suppression.html)
+- [ライト調整](https://shiguredo.github.io/media-processors/examples/light-adjustment)
 
 ## 優先実装
 
@@ -45,9 +48,6 @@ GitHub Pages にサンプルを用意しています。完全にクライアン
 **詳細は Discord やメールなどでお気軽にお問い合わせください**
 
 - フェイスフレーミング
-- 露出補正
-- ホワイトバランス編集
-- エコーキャンセル
 
 すでに存在するライブラリや仕組みを利用する前提となります。
 
@@ -72,8 +72,8 @@ Discord へお願いします。
 [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)
 
 ```
-Copyright 2022-2022, Takeru Ohta (Original Author)
-Copyright 2022-2022, Shiguredo Inc.
+Copyright 2022-2023, Takeru Ohta (Original Author)
+Copyright 2022-2023, Shiguredo Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

diff --git a/examples/video-multi-processors.html b/examples/video-multi-processors.html
@@ -0,0 +1,42 @@
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>Media Processors: ライト調整 + 仮想背景</title>
+  </head>
+  <body>
+    <h1>Media Processors: ライト調整 + 仮想背景</h1>
+
+    <h2>左: 処理後の映像、右: オリジナル映像</h2>    
+    <video id="processedVideo" muted autoplay playsinline></video>
+    <video id="originalVideo" muted autoplay playsinline></video>
+
+    <script src="./light-adjustment/light_adjustment.js"></script>
+    <script src="./virtual-background/virtual_background.js"></script>
+    <script>
+      if (!Shiguredo.VirtualBackgroundProcessor.isSupported() || !Shiguredo.LightAdjustmentProcessor.isSupported()) {
+        alert("Unsupported platform");
+        throw Error("Unsupported platform");
+      }
+
+      function getUserMedia() {
+        const constraints = {
+          width: 320,
+          height: 240,
+        };
+        return navigator.mediaDevices.getUserMedia({video: constraints});
+      }
+
+      getUserMedia().then(async (stream) => {
+        const lightAdjustmentProcessor = new Shiguredo.LightAdjustmentProcessor();
+        const virtualBackgroundProcessor = new Shiguredo.VirtualBackgroundProcessor("./virtual-background/");
+
+        const originalTrack = stream.getVideoTracks()[0];
+        const processedTrack0 = await lightAdjustmentProcessor.startProcessing(originalTrack);
+        const processedTrack1 = await virtualBackgroundProcessor.startProcessing(processedTrack0, {blurRadius: 5});
+
+        document.getElementById("originalVideo").srcObject = new MediaStream([originalTrack]);
+        document.getElementById("processedVideo").srcObject = new MediaStream([processedTrack1]);
+      });
+    </script>
+  </body>
+</html>
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/light-adjustment/package.json b/packages/light-adjustment/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@shiguredo/light-adjustment",
-  "version": "2023.1.0",
+  "version": "2023.2.0",
   "description": "Light Adjustment Library",
   "author": "Shiguredo Inc.",
   "license": "Apache-2.0",

diff --git a/packages/light-adjustment/src/light_adjustment.ts b/packages/light-adjustment/src/light_adjustment.ts
@@ -325,10 +325,8 @@ class WasmLightAdjustment {
     this.wasm = wasm;
     this.memory = wasm.exports.memory as WebAssembly.Memory;
 
-    const { width, height } = track.getSettings();
-    if (width === undefined || height === undefined) {
-      throw new Error("Failed to get video track resolution");
-    }
+    const width = track.getSettings().width || 0;
+    const height = track.getSettings().height || 0;
     this.imageWidth = width;
     this.imageHeight = height;
 

diff --git a/packages/video-track-processor/src/video_track_processor.ts b/packages/video-track-processor/src/video_track_processor.ts
@@ -97,10 +97,8 @@ class BreakoutBoxProcessor extends Processor {
     this.processor = new MediaStreamTrackProcessor({ track: this.track });
 
     // 作業用キャンバスを作成
-    const { width, height } = track.getSettings();
-    if (width === undefined || height === undefined) {
-      throw Error(`Could not retrieve the resolution of the video track: {track}`);
-    }
+    const width = track.getSettings().width || 0;
+    const height = track.getSettings().height || 0;
 
     this.canvas = new OffscreenCanvas(width, height);
     const canvasCtx = this.canvas.getContext("2d", { desynchronized: true, willReadFrequently: true });
@@ -170,9 +168,15 @@ class BreakoutBoxProcessor extends Processor {
 
 class RequestVideoFrameCallbackProcessor extends Processor {
   private video: HTMLVideoElement;
+  private requestVideoFrameCallbackHandle?: number;
+
+  // 処理結果画像を書き込むキャンバス
   private canvas: HTMLCanvasElement;
   private canvasCtx: CanvasRenderingContext2D;
-  private requestVideoFrameCallbackHandle?: number;
+
+  // 処理途中の作業用キャンバス
+  private tmpCanvas: OffscreenCanvas;
+  private tmpCanvasCtx: OffscreenCanvasRenderingContext2D;
 
   constructor(track: MediaStreamVideoTrack, callback: ProcessImageDataCallback) {
     super(track, callback);
@@ -185,13 +189,9 @@ class RequestVideoFrameCallbackProcessor extends Processor {
     this.video.srcObject = new MediaStream([track]);
 
     // 処理後の映像フレームを書き込むための canvas を生成する
-    //
-    // captureStream() メソッドは OffscreenCanvas にはないようなので HTMLCanvasElement を使用する
-    const width = track.getSettings().width;
-    const height = track.getSettings().height;
-    if (width === undefined || height === undefined) {
-      throw Error(`Could not retrieve the resolution of the video track: {track}`);
-    }
+    // captureStream() を使いたいので OffscreenCanvas にはできない
+    const width = track.getSettings().width || 0;
+    const height = track.getSettings().height || 0;
     this.canvas = document.createElement("canvas");
     this.canvas.width = width;
     this.canvas.height = height;
@@ -200,6 +200,14 @@ class RequestVideoFrameCallbackProcessor extends Processor {
       throw Error("Failed to create 2D canvas context");
     }
     this.canvasCtx = canvasCtx;
+
+    // 作業用キャンバスを生成する
+    this.tmpCanvas = createOffscreenCanvas(width, height) as OffscreenCanvas;
+    const tmpCanvasCtx = this.tmpCanvas.getContext("2d");
+    if (tmpCanvasCtx === null) {
+      throw Error("Failed to create 2D canvas context");
+    }
+    this.tmpCanvasCtx = tmpCanvasCtx;
   }
 
   static isSupported(): boolean {
@@ -227,9 +235,10 @@ class RequestVideoFrameCallbackProcessor extends Processor {
   private async onFrame() {
     const { videoWidth: width, videoHeight: height } = this.video;
     resizeCanvasIfNeed(width, height, this.canvas);
+    resizeCanvasIfNeed(width, height, this.tmpCanvas);
 
-    this.canvasCtx.drawImage(this.video, 0, 0);
-    const image = this.canvasCtx.getImageData(0, 0, width, height);
+    this.tmpCanvasCtx.drawImage(this.video, 0, 0);
+    const image = this.tmpCanvasCtx.getImageData(0, 0, width, height);
     const processedImage = await this.callback(image);
     this.canvasCtx.putImageData(processedImage, 0, 0);
 
@@ -241,6 +250,19 @@ class RequestVideoFrameCallbackProcessor extends Processor {
   }
 }
 
+// TODO(sile): Safari 16.4 から OffscreenCanvas に対応したので、そのうちにこの関数は削除する
+function createOffscreenCanvas(width: number, height: number): OffscreenCanvas | HTMLCanvasElement {
+  if (typeof OffscreenCanvas === "undefined") {
+    // OffscreenCanvas が使えない場合には通常の canvas で代替する
+    const canvas = document.createElement("canvas");
+    canvas.width = width;
+    canvas.height = height;
+    return canvas;
+  } else {
+    return new OffscreenCanvas(width, height);
+  }
+}
+
 function resizeCanvasIfNeed(width: number, height: number, canvas: OffscreenCanvas | HTMLCanvasElement) {
   if (canvas.width !== width || canvas.height !== height) {
     canvas.width = width;

diff --git a/packages/virtual-background/src/virtual_background.ts b/packages/virtual-background/src/virtual_background.ts
@@ -153,12 +153,9 @@ class VirtualBackgroundProcessor {
     track: MediaStreamVideoTrack,
     options: VirtualBackgroundProcessorOptions = {}
   ): Promise<MediaStreamVideoTrack> {
-    const width = track.getSettings().width;
-    const height = track.getSettings().height;
-    if (width === undefined || height === undefined) {
-      throw Error(`Could not retrieve the resolution of the video track: {track}`);
-    }
-    const canvas = createOffscreenCanvas(width, height);
+    const initialWidth = track.getSettings().width || 0;
+    const initialHeight = track.getSettings().height || 0;
+    const canvas = createOffscreenCanvas(initialWidth, initialHeight);
     const canvasCtx = canvas.getContext("2d", {
       desynchronized: true,
       willReadFrequently: true,
@@ -172,7 +169,7 @@ class VirtualBackgroundProcessor {
     // TODO(sile): Safari が filter に対応したらこの分岐は削除する
     let blurCanvasCtx: OffscreenCanvasRenderingContext2D | undefined;
     if (options.blurRadius !== undefined && browser() === "safari") {
-      const ctx = createOffscreenCanvas(width, height).getContext("2d", {
+      const ctx = createOffscreenCanvas(initialWidth, initialHeight).getContext("2d", {
         desynchronized: true,
         willReadFrequently: true,
       });
@@ -190,6 +187,7 @@ class VirtualBackgroundProcessor {
     }
     this.segmentation.setOptions({ modelSelection });
     this.segmentation.onResults((results) => {
+      const { width, height } = results.segmentationMask;
       resizeCanvasIfNeed(width, height, canvas);
       if (blurCanvasCtx !== undefined) {
         resizeCanvasIfNeed(width, height, blurCanvasCtx.canvas);
@@ -328,6 +326,7 @@ function trimLastSlash(s: string): string {
   return s;
 }
 
+// TODO(sile): Safari 16.4 から OffscreenCanvas に対応したので、そのうちにこの関数は削除する
 function createOffscreenCanvas(width: number, height: number): OffscreenCanvas | HTMLCanvasElement {
   if (typeof OffscreenCanvas === "undefined") {
     // OffscreenCanvas が使えない場合には通常の canvas で代替する