Skip to content

Commit

Permalink
Merge branch 'release/light-adjustment-gpu-2023.1.1'
Browse files Browse the repository at this point in the history
  • Loading branch information
sile committed Jan 19, 2024
2 parents ba12f44 + 0d5f06c commit 33cd005
Show file tree
Hide file tree
Showing 53 changed files with 3,594 additions and 2,513 deletions.
30 changes: 19 additions & 11 deletions .github/renovate.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,33 @@
"schedule:weekly"
],
"dependencyDashboard": false,
"ignorePaths": [
"**/packages/light-adjustment-gpu/model/**"
],
"packageRules": [
{
"matchUpdateTypes": ["minor", "patch", "pin", "digest"],
"platformAutomerge": true,
"automerge": true
},
{
"matchUpdateTypes": ["minor", "patch", "pin", "digest"],
"matchPackagePatterns": ["eslint"],
"groupName": "eslint",
"matchUpdateTypes": [
"minor",
"patch",
"pin",
"digest"
],
"platformAutomerge": true,
"automerge": true
},
{
"matchUpdateTypes": ["minor", "patch", "pin", "digest"],
"matchPackagePatterns": ["rollup"],
"matchUpdateTypes": [
"minor",
"patch",
"pin",
"digest"
],
"matchPackagePatterns": [
"rollup"
],
"groupName": "rollup",
"platformAutomerge": true,
"automerge": true
}
]
}
}
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
- uses: actions/checkout@v4
- uses: goto-bus-stop/setup-zig@v2
- run: zig version
- uses: actions/setup-node@v3
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node }}
- uses: pnpm/action-setup@v2
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
Expand All @@ -53,7 +53,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
uses: github/codeql-action/autobuild@v3

# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
Expand All @@ -67,4 +67,4 @@ jobs:
# make release

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@v3
4 changes: 2 additions & 2 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
- uses: actions/checkout@v4
- uses: goto-bus-stop/setup-zig@v2
- run: zig version
- uses: actions/setup-node@v3
- uses: actions/setup-node@v4
with:
node-version: "18"
- uses: pnpm/action-setup@v2
Expand Down Expand Up @@ -60,4 +60,4 @@ jobs:
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v2
uses: actions/deploy-pages@v3
3 changes: 3 additions & 0 deletions .vscode/extensions.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"recommendations": ["biomejs.biome"]
}
20 changes: 20 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode",
"[javascript]": {
"editor.defaultFormatter": "biomejs.biome"
},
"[javascriptreact]": {
"editor.defaultFormatter": "biomejs.biome"
},
"[typescript]": {
"editor.defaultFormatter": "biomejs.biome"
},
"[typescriptreact]": {
"editor.defaultFormatter": "biomejs.biome"
},
"editor.codeActionsOnSave": {
"quickfix.biome": "explicit",
"source.organizeImports.biome": "explicit"
}
}
35 changes: 24 additions & 11 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,24 +11,37 @@

## develop

- [FIX] Zig の組み込みのキャスト関数xToYがyFromXにリネームされたので対応
- [FIX] Zig の WebAssembly ビルドで addSharedLibrary から addBinary を使うよう変更になったので対応
- https://ziglang.org/documentation/master/#Freestanding
- https://github.com/ziglang/zig/pull/17815
- [FIX] zig で std.mem.copy ではなく @memcpy を使うようにする
- https://github.com/ziglang/zig/pull/18143
- @voluntas
- [FIX] Zig の組み込みのキャスト関数 xToY が yFromX にリネームされたので対応
- @kounoike
- [FIX] Zig の `@truncate` やキャスト関連に破壊的変更があったため対応
- @kounoike

## light-adjustment-gpu-2023.1.1

- [FIX] ライト調整 GPU 版の semantic_guided_llie_324x240 のパラメータが間違っていたのを修正
- @kounoike
- [FIX] image-to-image-video-processor だけ TypeScript 5.2.2 に固定する
- @voluntas

## light-adjustment-gpu-2023.1.0

**初リリース**

- [ADD] 推論ベースの Low-Light-Image-Enhancement を行う @shiguredo/light-adjustment-gpu パッケージを追加
- 暗い画像に対してコントラストを改善したり、逆光で暗くなっている人物を明るくすることができる
- 内部的には @shiguredo/image-to-image-video-processor を利用する
- https://github.com/PINTO0309/PINTO_model_zoo/tree/main/370_Semantic-Guided-Low-Light-Image-Enhancement のモデルをtfjsに変換して使用する
- https://github.com/PINTO0309/PINTO_model_zoo/tree/main/370_Semantic-Guided-Low-Light-Image-Enhancement のモデルを tfjs に変換して使用する
- @kounoike
- [ADD] 画像画像の推論を行う際に使える @shiguredo/image-to-image-video-processor パッケージを追加する
- Tensorflow.jsを使い、画像画像の何らかの推論を行う
- [ADD] 画像画像の推論を行う際に使える @shiguredo/image-to-image-video-processor パッケージを追加する
- Tensorflow.js を使い、画像画像の何らかの推論を行う
- 推論前後のリサイズ処理を入力解像度に合わせて自動的に行う
- Low Light Image Enhancementのような明るさ補正、超解像、ノイズ除去などの推論で使われることを想定している
- Low Light Image Enhancement のような明るさ補正、超解像、ノイズ除去などの推論で使われることを想定している
- @kounoike
- [CHANGE] tsconfig.json の compilerOptions の lib を es2015 から es2020 に変更する
- @voluntas
Expand All @@ -42,7 +55,7 @@

## virtual-background-2023.2.0

- [UPDATE] CPU-GPUの転送を減らして背景ぼかし処理を高速化
- [UPDATE] CPU-GPU の転送を減らして背景ぼかし処理を高速化
- @kounoike
- [UPDATE] GitHub Actions に Node 20 を追加
- @voluntas
Expand Down Expand Up @@ -118,30 +131,30 @@

## noise-suppression-2022.4.1

- [FIX] 公開APIに含まれる型を定義しているパッケージをdependenciesに追加
- [FIX] 公開 API に含まれる型を定義しているパッケージを dependencies に追加
- https://github.com/shiguredo/media-processors/pull/26
- @sile

## virtual-background-2022.4.1

- [FIX] 公開APIに含まれる型を定義しているパッケージをdependenciesに追加
- [FIX] 公開 API に含まれる型を定義しているパッケージを dependencies に追加
- https://github.com/shiguredo/media-processors/pull/26
- @sile

## noise-suppression-2022.4.0

- [ADD] RNNoiseモデルの差し替えに対応
- [ADD] RNNoise モデルの差し替えに対応

- https://github.com/shiguredo/media-processors/pull/15
- @sile

- [ADD] 処理適用前後のメディアトラックを取得するためのAPIを追加
- [ADD] 処理適用前後のメディアトラックを取得するための API を追加
- https://github.com/shiguredo/media-processors/pull/25
- @sile

## virtual-background-2022.4.0

- [ADD] 処理適用前後のメディアトラックを取得するためのAPIを追加
- [ADD] 処理適用前後のメディアトラックを取得するための API を追加
- https://github.com/shiguredo/media-processors/pull/25
- @sile

Expand Down
12 changes: 10 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,27 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use

## 方針

- 動作するブラウザは最新の Chrome または Edge のみです。それ以外のブラウザでは動作しません
- iOS の Chrome や Edge では動作しません
対応ブラウザは最新の Chrome または Edge です。それ以外のブラウザでは動作しません。

ただし、以下の機能は Safari でも動作します。

- 仮想背景 / 背景ぼかし
- ライト調整
- ライト調整(GPU版)

## 機能

- [仮想背景 / 背景ぼかし](https://github.com/shiguredo/media-processors/tree/develop/packages/virtual-background)
- [ノイズ抑制](https://github.com/shiguredo/media-processors/tree/develop/packages/noise-suppression)
- [ライト調整](https://github.com/shiguredo/media-processors/tree/develop/packages/light-adjustment)
- [ライト調整(GPU版)](https://github.com/shiguredo/media-processors/tree/develop/packages/light-adjustment-gpu)

## npm

- [@shiguredo/virtual\-background \- npm](https://www.npmjs.com/package/@shiguredo/virtual-background)
- [@shiguredo/noise\-suppression \- npm](https://www.npmjs.com/package/@shiguredo/noise-suppression)
- [@shiguredo/light\-adjustment \- npm](https://www.npmjs.com/package/@shiguredo/light-adjustment)
- [@shiguredo/light\-adjustment\-gpu \- npm](https://www.npmjs.com/package/@shiguredo/light-adjustment-gpu)

## サンプル

Expand All @@ -38,6 +45,7 @@ GitHub Pages にサンプルを用意しています。完全にクライアン
- [仮想背景 / 背景ぼかし](https://shiguredo.github.io/media-processors/examples/virtual-background.html)
- [ノイズ抑制](https://shiguredo.github.io/media-processors/examples/noise-suppression.html)
- [ライト調整](https://shiguredo.github.io/media-processors/examples/light-adjustment)
- [ライト調整(GPU版)](https://shiguredo.github.io/media-processors/examples/light-adjustment-gpu)

## 優先実装

Expand Down
45 changes: 45 additions & 0 deletions biome.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"$schema": "https://biomejs.dev/schemas/1.4.0/schema.json",
"organizeImports": {
"enabled": true
},
"linter": {
"enabled": true,
"rules": {
"recommended": true
}
},
"formatter": {
"enabled": true,
"formatWithErrors": false,
"ignore": [],
"indentStyle": "space",
"indentWidth": 2,
"lineWidth": 100
},
"json": {
"parser": {
"allowComments": true
},
"formatter": {
"enabled": true,
"indentStyle": "space",
"indentWidth": 2,
"lineWidth": 100
}
},
"javascript": {
"formatter": {
"enabled": true,
"quoteStyle": "single",
"jsxQuoteStyle": "double",
"trailingComma": "all",
"semicolons": "asNeeded",
"arrowParentheses": "always",
"indentStyle": "space",
"indentWidth": 2,
"lineWidth": 100,
"quoteProperties": "asNeeded"
}
}
}
55 changes: 55 additions & 0 deletions examples/video-multi-processors-gpu.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
<html>
<head>
<meta charset="utf-8">
<title>Media Processors: ライト調整(GPU) + 仮想背景</title>
</head>
<h1>Media Processors: ライト調整(GPU) + 仮想背景</h1>

<h2>左: 処理後の映像、右: オリジナル映像</h2>
ライト調整: フレーム処理時間: <span id="elapsed0">0.0</span> 秒 フレームレート: <span id="fps0">0.0</span>FPS<br />
仮想背景: フレーム処理時間: <span id="elapsed1">0.0</span> 秒 フレームレート: <span id="fps1">0.0</span>FPS<br />
<video id="processedVideo" muted autoplay playsinline></video>
<video id="originalVideo" muted autoplay playsinline></video>

<script src="./light-adjustment-gpu/light_adjustment_gpu.js"></script>
<script src="./virtual-background/virtual_background.js"></script>
<script>
if (!Shiguredo.VirtualBackgroundProcessor.isSupported() || !Shiguredo.LightAdjustmentGpuProcessor.isSupported()) {
alert("Unsupported platform");
throw Error("Unsupported platform");
}
const lightAdjustmentProcessor = new Shiguredo.LightAdjustmentGpuProcessor("./light-adjustment-gpu/", "semantic_guided_llie_648x480", 0.5);
const virtualBackgroundProcessor = new Shiguredo.VirtualBackgroundProcessor("./virtual-background/");
setInterval(() => {
const elapsed0 = lightAdjustmentProcessor.getAverageProcessedTimeMs() / 1000;
document.getElementById('elapsed0').innerText = elapsed0.toFixed(4).padStart(4, '0');
const fps0 = lightAdjustmentProcessor.getFps();
document.getElementById('fps0').innerText = fps0.toFixed(2).padStart(5, '0');
const elapsed1 = virtualBackgroundProcessor.getAverageProcessedTimeMs() / 1000;
document.getElementById('elapsed1').innerText = elapsed1.toFixed(4).padStart(4, '0');
const fps1 = virtualBackgroundProcessor.getFps();
document.getElementById('fps1').innerText = fps1.toFixed(2).padStart(5, '0');
}, 300);

function getUserMedia() {
const constraints = {
width: 320,
height: 240,
};
return navigator.mediaDevices.getUserMedia({video: constraints});
}

getUserMedia().then(async (stream) => {
const backgroundImage = new Image();
backgroundImage.src = 'background.jpg';

const originalTrack = stream.getVideoTracks()[0];
const processedTrack0 = await lightAdjustmentProcessor.startProcessing(originalTrack);
const processedTrack1 = await virtualBackgroundProcessor.startProcessing(processedTrack0, {backgroundImage});

document.getElementById("originalVideo").srcObject = new MediaStream([originalTrack]);
document.getElementById("processedVideo").srcObject = new MediaStream([processedTrack1]);
});
</script>
</body>
</html>
Loading

0 comments on commit 33cd005

Please sign in to comment.