From ab075280ff84612b498a246e31d6090626128f03 Mon Sep 17 00:00:00 2001 From: Matt Hillsdon <44397098+microbit-matt-hillsdon@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:39:21 +0100 Subject: [PATCH] Resample via a libsamplerate (#117) We need to do this for recording as we can't specify a rate. We need to do this for playback for sample rates outside of the browser's supported range. That includes the current default rate for Firefox and older Safari. Uses a resampler build that doesn't support medium/best to save on bundle size. I've also increased the buffer size so I can play a 44k sample in an OK-ish way. Maybe 256 will be needed. We know Firefox on Windows still performs poorly. It did badly with audio before these changes. Another significant issue is that it can take 3 seconds to call getUserMedia on Desktop Safari. --- README.md | 20 ++ package-lock.json | 475 ++++++++++++++++++++++++++++-- package.json | 3 + src/board/audio/index.ts | 110 ++++--- src/board/conversions.ts | 7 +- src/board/index.ts | 13 + src/demo.html | 1 + src/examples/record.py | 3 +- src/examples/record_background.py | 19 ++ src/jshal.js | 18 +- src/mpconfigport.h | 3 +- 11 files changed, 573 insertions(+), 99 deletions(-) create mode 100644 src/examples/record_background.py diff --git a/README.md b/README.md index 091bccde..40ebdaf0 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,26 @@ Steps for WASM debugging in Chrome: - Enable "WebAssembly Debugging: Enable DWARF support" in DevTools Experiments - DEBUG=1 make +## License + +This software is under the MIT open source license. + +[SPDX-License-Identifier: MIT](LICENSE) + +MicroPython for micro:bit is included in the build process via a submodule. + +We use dependencies via the NPM registry as specified by the package.json file under common Open Source licenses. + +Full details of each package can be found by running `license-checker`: + +```bash +$ npx license-checker --direct --summary --production +``` + +Omit the flags as desired to obtain more detail. + +A fork of libsamplerate_js to reduce bundle size is [hosted on GitHub](https://github.com/microbit-foundation/libsamplerate-js). + ## Code of Conduct Trust, partnership, simplicity and passion are our core values we live and diff --git a/package-lock.json b/package-lock.json index 8fab2c2a..67484c9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "@microbit-foundation/microbit-micropython-v2-simulator", "version": "0.1.0", "license": "MIT", + "dependencies": { + "@alexanderolsen/libsamplerate-js": "microbit-foundation/libsamplerate-js#v2.1.2-microbit.1" + }, "devDependencies": { "@types/emscripten": "^1.39.10", "esbuild": "^0.14.49", @@ -15,6 +18,43 @@ "vitest": "^0.22.1" } }, + "node_modules/@alexanderolsen/libsamplerate-js": { + "version": "2.1.2-microbit.1", + "resolved": "git+ssh://git@github.com/microbit-foundation/libsamplerate-js.git#14bdda3603133d21c07cfb48974617504576a7f7", + "license": "MIT" + }, + "node_modules/@esbuild/android-arm": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.18.tgz", + "integrity": "sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.15.18.tgz", + "integrity": "sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@types/chai": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.3.tgz", @@ -463,9 +503,9 @@ } }, "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "dev": true, "hasInstallScript": true, "optional": true, @@ -483,9 +523,9 @@ "dev": true }, "node_modules/get-func-name": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", + "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", "dev": true, "engines": { "node": "*" @@ -544,10 +584,16 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", - "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -571,15 +617,15 @@ } }, "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==", "dev": true }, "node_modules/postcss": { - "version": "8.4.16", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.16.tgz", - "integrity": "sha512-ipHE1XBvKzm5xI7hiHCZJCSugxvsdq2mPnsq5+UF+VHCjiBvtDrlxJfMBToWaP9D5XlgNmcFGqoHmUn0EYEaRQ==", + "version": "8.4.41", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", + "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==", "dev": true, "funding": [ { @@ -589,12 +635,16 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ], "dependencies": { - "nanoid": "^3.3.4", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" + "nanoid": "^3.3.7", + "picocolors": "^1.0.1", + "source-map-js": "^1.2.0" }, "engines": { "node": "^10 || ^12 || >=14" @@ -633,9 +683,9 @@ } }, "node_modules/rollup": { - "version": "2.77.3", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.77.3.tgz", - "integrity": "sha512-/qxNTG7FbmefJWoeeYJFbHehJ2HNWnjkAFRKzWN/45eNBBF/r8lo992CwcJXEzyVxs5FmfId+vTSTQDb+bxA+g==", + "version": "2.79.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.1.tgz", + "integrity": "sha512-uKxbd0IhMZOhjAiD5oAFp7BqvkA4Dv47qpOCtaNvng4HBwdbWtdOh8f5nZNuk2rp51PMGk3bzfWu5oayNEuYnw==", "dev": true, "bin": { "rollup": "dist/bin/rollup" @@ -648,9 +698,9 @@ } }, "node_modules/source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", "dev": true, "engines": { "node": ">=0.10.0" @@ -696,15 +746,15 @@ } }, "node_modules/vite": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-3.0.9.tgz", - "integrity": "sha512-waYABTM+G6DBTCpYAxvevpG50UOlZuynR0ckTK5PawNVt7ebX6X7wNXHaGIO6wYYFXSM7/WcuFuO2QzhBB6aMw==", + "version": "3.2.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-3.2.10.tgz", + "integrity": "sha512-Dx3olBo/ODNiMVk/cA5Yft9Ws+snLOXrhLtrI3F4XLt4syz2Yg8fayZMWScPKoz12v5BUv7VEmQHnsfpY80fYw==", "dev": true, "dependencies": { - "esbuild": "^0.14.47", - "postcss": "^8.4.16", + "esbuild": "^0.15.9", + "postcss": "^8.4.18", "resolve": "^1.22.1", - "rollup": ">=2.75.6 <2.77.0 || ~2.77.0" + "rollup": "^2.79.1" }, "bin": { "vite": "bin/vite.js" @@ -716,12 +766,17 @@ "fsevents": "~2.3.2" }, "peerDependencies": { + "@types/node": ">= 14", "less": "*", "sass": "*", "stylus": "*", + "sugarss": "*", "terser": "^5.4.0" }, "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, "less": { "optional": true }, @@ -731,11 +786,371 @@ "stylus": { "optional": true }, + "sugarss": { + "optional": true + }, "terser": { "optional": true } } }, + "node_modules/vite/node_modules/esbuild": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.18.tgz", + "integrity": "sha512-x/R72SmW3sSFRm5zrrIjAhCeQSAWoni3CmHEqfQrZIQTM3lVCdehdwuIqaOtfC2slvpdlLa62GYoN8SxT23m6Q==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.15.18", + "@esbuild/linux-loong64": "0.15.18", + "esbuild-android-64": "0.15.18", + "esbuild-android-arm64": "0.15.18", + "esbuild-darwin-64": "0.15.18", + "esbuild-darwin-arm64": "0.15.18", + "esbuild-freebsd-64": "0.15.18", + "esbuild-freebsd-arm64": "0.15.18", + "esbuild-linux-32": "0.15.18", + "esbuild-linux-64": "0.15.18", + "esbuild-linux-arm": "0.15.18", + "esbuild-linux-arm64": "0.15.18", + "esbuild-linux-mips64le": "0.15.18", + "esbuild-linux-ppc64le": "0.15.18", + "esbuild-linux-riscv64": "0.15.18", + "esbuild-linux-s390x": "0.15.18", + "esbuild-netbsd-64": "0.15.18", + "esbuild-openbsd-64": "0.15.18", + "esbuild-sunos-64": "0.15.18", + "esbuild-windows-32": "0.15.18", + "esbuild-windows-64": "0.15.18", + "esbuild-windows-arm64": "0.15.18" + } + }, + "node_modules/vite/node_modules/esbuild-android-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.15.18.tgz", + "integrity": "sha512-wnpt3OXRhcjfIDSZu9bnzT4/TNTDsOUvip0foZOUBG7QbSt//w3QV4FInVJxNhKc/ErhUxc5z4QjHtMi7/TbgA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-android-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.15.18.tgz", + "integrity": "sha512-G4xu89B8FCzav9XU8EjsXacCKSG2FT7wW9J6hOc18soEHJdtWu03L3TQDGf0geNxfLTtxENKBzMSq9LlbjS8OQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-darwin-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.18.tgz", + "integrity": "sha512-2WAvs95uPnVJPuYKP0Eqx+Dl/jaYseZEUUT1sjg97TJa4oBtbAKnPnl3b5M9l51/nbx7+QAEtuummJZW0sBEmg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-darwin-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.15.18.tgz", + "integrity": "sha512-tKPSxcTJ5OmNb1btVikATJ8NftlyNlc8BVNtyT/UAr62JFOhwHlnoPrhYWz09akBLHI9nElFVfWSTSRsrZiDUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-freebsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.15.18.tgz", + "integrity": "sha512-TT3uBUxkteAjR1QbsmvSsjpKjOX6UkCstr8nMr+q7zi3NuZ1oIpa8U41Y8I8dJH2fJgdC3Dj3CXO5biLQpfdZA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-freebsd-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.15.18.tgz", + "integrity": "sha512-R/oVr+X3Tkh+S0+tL41wRMbdWtpWB8hEAMsOXDumSSa6qJR89U0S/PpLXrGF7Wk/JykfpWNokERUpCeHDl47wA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-32": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.15.18.tgz", + "integrity": "sha512-lphF3HiCSYtaa9p1DtXndiQEeQDKPl9eN/XNoBf2amEghugNuqXNZA/ZovthNE2aa4EN43WroO0B85xVSjYkbg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.15.18.tgz", + "integrity": "sha512-hNSeP97IviD7oxLKFuii5sDPJ+QHeiFTFLoLm7NZQligur8poNOWGIgpQ7Qf8Balb69hptMZzyOBIPtY09GZYw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-arm": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.15.18.tgz", + "integrity": "sha512-UH779gstRblS4aoS2qpMl3wjg7U0j+ygu3GjIeTonCcN79ZvpPee12Qun3vcdxX+37O5LFxz39XeW2I9bybMVA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.15.18.tgz", + "integrity": "sha512-54qr8kg/6ilcxd+0V3h9rjT4qmjc0CccMVWrjOEM/pEcUzt8X62HfBSeZfT2ECpM7104mk4yfQXkosY8Quptug==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-mips64le": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.15.18.tgz", + "integrity": "sha512-Mk6Ppwzzz3YbMl/ZZL2P0q1tnYqh/trYZ1VfNP47C31yT0K8t9s7Z077QrDA/guU60tGNp2GOwCQnp+DYv7bxQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-ppc64le": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.15.18.tgz", + "integrity": "sha512-b0XkN4pL9WUulPTa/VKHx2wLCgvIAbgwABGnKMY19WhKZPT+8BxhZdqz6EgkqCLld7X5qiCY2F/bfpUUlnFZ9w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-riscv64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.15.18.tgz", + "integrity": "sha512-ba2COaoF5wL6VLZWn04k+ACZjZ6NYniMSQStodFKH/Pu6RxzQqzsmjR1t9QC89VYJxBeyVPTaHuBMCejl3O/xg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-s390x": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.15.18.tgz", + "integrity": "sha512-VbpGuXEl5FCs1wDVp93O8UIzl3ZrglgnSQ+Hu79g7hZu6te6/YHgVJxCM2SqfIila0J3k0csfnf8VD2W7u2kzQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-netbsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.15.18.tgz", + "integrity": "sha512-98ukeCdvdX7wr1vUYQzKo4kQ0N2p27H7I11maINv73fVEXt2kyh4K4m9f35U1K43Xc2QGXlzAw0K9yoU7JUjOg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-openbsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.15.18.tgz", + "integrity": "sha512-yK5NCcH31Uae076AyQAXeJzt/vxIo9+omZRKj1pauhk3ITuADzuOx5N2fdHrAKPxN+zH3w96uFKlY7yIn490xQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-sunos-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.18.tgz", + "integrity": "sha512-On22LLFlBeLNj/YF3FT+cXcyKPEI263nflYlAhz5crxtp3yRG1Ugfr7ITyxmCmjm4vbN/dGrb/B7w7U8yJR9yw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-windows-32": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.15.18.tgz", + "integrity": "sha512-o+eyLu2MjVny/nt+E0uPnBxYuJHBvho8vWsC2lV61A7wwTWC3jkN2w36jtA+yv1UgYkHRihPuQsL23hsCYGcOQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-windows-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.15.18.tgz", + "integrity": "sha512-qinug1iTTaIIrCorAUjR0fcBk24fjzEedFYhhispP8Oc7SFvs+XeW3YpAKiKp8dRpizl4YYAhxMjlftAMJiaUw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-windows-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.15.18.tgz", + "integrity": "sha512-q9bsYzegpZcLziq0zgUi5KqGVtfhjxGbnksaBFYmWLxeV/S1fK4OLdq2DFYnXcLMjlZw2L0jLsk1eGoB522WXQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/vitest": { "version": "0.22.1", "resolved": "https://registry.npmjs.org/vitest/-/vitest-0.22.1.tgz", diff --git a/package.json b/package.json index 5856f89c..2bee2d66 100644 --- a/package.json +++ b/package.json @@ -28,5 +28,8 @@ "esbuild": "^0.14.49", "prettier": "2.6.0", "vitest": "^0.22.1" + }, + "dependencies": { + "@alexanderolsen/libsamplerate-js": "microbit-foundation/libsamplerate-js#v2.1.2-microbit.1" } } diff --git a/src/board/audio/index.ts b/src/board/audio/index.ts index 9dd127bf..1fdb67e2 100644 --- a/src/board/audio/index.ts +++ b/src/board/audio/index.ts @@ -1,6 +1,11 @@ +import { SRC } from "@alexanderolsen/libsamplerate-js/dist/src"; import { replaceBuiltinSound } from "./built-in-sounds"; import { SoundEmojiSynthesizer } from "./sound-emoji-synthesizer"; import { parseSoundEffects } from "./sound-expressions"; +import { + create as createSampleRateConverter, + ConverterType, +} from "@alexanderolsen/libsamplerate-js"; declare global { interface Window { @@ -11,7 +16,11 @@ declare global { interface AudioOptions { defaultAudioCallback: () => void; + defaultResampler: SRC; speechAudioCallback: () => void; + speechResampler: SRC; + soundExpressionResampler: SRC; + recordingResampler: SRC; } export class BoardAudio { @@ -24,6 +33,8 @@ export class BoardAudio { private muteNode: GainNode | undefined; private sensitivityNode: GainNode | undefined; + private recordingResampler: SRC | undefined; + default: BufferedAudio | undefined; speech: BufferedAudio | undefined; soundExpression: BufferedAudio | undefined; @@ -34,11 +45,17 @@ export class BoardAudio { initializeCallbacks({ defaultAudioCallback, + defaultResampler, speechAudioCallback, + speechResampler, + soundExpressionResampler, + recordingResampler, }: AudioOptions) { if (!this.context) { throw new Error("Context must be pre-created from a user event"); } + this.recordingResampler = recordingResampler; + this.muteNode = this.context.createGain(); this.muteNode.gain.setValueAtTime( this.muted ? 0 : 1, @@ -56,16 +73,19 @@ export class BoardAudio { this.default = new BufferedAudio( this.context, this.volumeNode, + defaultResampler, defaultAudioCallback ); this.speech = new BufferedAudio( this.context, this.volumeNode, + speechResampler, speechAudioCallback ); this.soundExpression = new BufferedAudio( this.context, this.volumeNode, + soundExpressionResampler, () => { if (this.currentSoundExpressionCallback) { this.currentSoundExpressionCallback(); @@ -75,12 +95,13 @@ export class BoardAudio { } async createAudioContextFromUserInteraction(): Promise { + // If we set a 44.1kHz rate then we fail to connect to user media on Mac as it selects 48000 + // So we leave it at the default hoping it's most likely to match user media... + // Until there's progress on this there doesn't seem a better way: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1674892 this.context = - this.context ?? - new (window.AudioContext || window.webkitAudioContext)({ - // The highest rate is the sound expression synth. - sampleRate: 44100, - }); + this.context ?? new (window.AudioContext || window.webkitAudioContext)(); + if (this.context.state === "suspended") { return this.context.resume(); } @@ -92,21 +113,16 @@ export class BoardAudio { this.stopSoundExpression(); }; const synth = new SoundEmojiSynthesizer(0, onDone); + this.soundExpression!.setSampleRate(synth.sampleRate); synth.play(soundEffects); const callback = () => { const source = synth.pull(); if (this.context) { - // Use createBuffer instead of new AudioBuffer to support Safari 14.0. - const target = this.context.createBuffer( - 1, - source.length, - synth.sampleRate - ); - const channel = target.getChannelData(0); + const target = new Float32Array(source.length); for (let i = 0; i < source.length; i++) { // Buffer is (0, 1023) we need to map it to (-1, 1) - channel[i] = (source[i] - 512) / 512; + target[i] = (source[i] - 512) / 512; } this.soundExpression!.writeData(target); } @@ -197,6 +213,7 @@ export class BoardAudio { try { micStream = await navigator.mediaDevices.getUserMedia({ video: false, + // It seems Firefox ignores the rate set here audio: true, }); } catch (e) { @@ -208,41 +225,34 @@ export class BoardAudio { const source = this.context!.createMediaStreamSource(micStream); source.connect(this.sensitivityNode!); - // TODO: consider AudioWorklet - worth it? Browser support? - // consider alternative resampling approaches - // what sample rates are actually supported this way? + const recorder = this.context!.createScriptProcessor(2048, 1, 1); + + const inputSampleRate = this.context!.sampleRate; + this.recordingResampler!.inputSampleRate = inputSampleRate; + this.recordingResampler!.outputSampleRate = sampleRate; + recorder.onaudioprocess = (e) => { - const offlineContext = new (window.OfflineAudioContext || - window.webkitOfflineAudioContext)( - 1, - sampleRate * (e.inputBuffer.length / e.inputBuffer.sampleRate), - sampleRate + const resampled = this.recordingResampler!.full( + e.inputBuffer.getChannelData(0) ); - const source = offlineContext.createBufferSource(); - source.buffer = e.inputBuffer; - source.connect(offlineContext.destination); - source.start(); - offlineContext.addEventListener("complete", (e) => { - onChunk(e.renderedBuffer.getChannelData(0)); - samplesSent += e.renderedBuffer.length; - if (samplesSent >= samplesNeeded) { - this.stopRecording(); - } - }); - offlineContext.startRendering(); + onChunk(resampled); + samplesSent += resampled.length; + if (samplesSent >= samplesNeeded) { + this.stopRecording(); + } }; - this.sensitivityNode!.connect(recorder); - recorder.connect(this.context!.destination); - this.stopActiveRecording = () => { recorder.disconnect(); this.sensitivityNode!.disconnect(); source.disconnect(); - micStream.getTracks().forEach((track) => track.stop()); + micStream?.getTracks().forEach((track) => track.stop()); this.microphoneEl.style.display = "none"; this.stopActiveRecording = undefined; }; + + this.sensitivityNode!.connect(recorder); + recorder.connect(this.context!.destination); } boardStopped() { @@ -263,31 +273,35 @@ export class BoardAudio { class BufferedAudio { nextStartTime: number = -1; - private sampleRate: number = -1; constructor( private context: AudioContext, private destination: AudioNode, + private resampler: SRC, private callback: () => void - ) {} + ) { + this.resampler.outputSampleRate = this.context.sampleRate; + } init(sampleRate: number) { // This is called for each new audio source so don't reset nextStartTime // or we start to overlap audio - this.sampleRate = sampleRate; - } - - createBuffer(length: number) { - // Use createBuffer instead of new AudioBuffer to support Safari 14.0. - return this.context.createBuffer(1, length, this.sampleRate); + this.setSampleRate(sampleRate); } setSampleRate(sampleRate: number) { - this.sampleRate = sampleRate; + this.resampler.inputSampleRate = sampleRate; } - writeData(buffer: AudioBuffer) { - // Use createBufferSource instead of new AudioBufferSourceNode to support Safari 14.0. + writeData(data: Float32Array) { + // In practice the supported range is less than the 8k..96k required by the spec and varies by browser + // for a consistent performance profile we're always resampling for now rather than letting Web Audio do it + let sampleRate = this.context.sampleRate; + data = this.resampler.full(data); + + // Use createXXX instead to support Safari 14.0. + const buffer = this.context.createBuffer(1, data.length, sampleRate); + buffer.copyToChannel(data, 0); const source = this.context.createBufferSource(); source.buffer = buffer; source.onended = this.callCallback; diff --git a/src/board/conversions.ts b/src/board/conversions.ts index 392d7808..f42bca7f 100644 --- a/src/board/conversions.ts +++ b/src/board/conversions.ts @@ -98,12 +98,11 @@ export function convertAccelerometerNumberToString(value: number): string { export const convertAudioBuffer = ( heap: Uint8Array, source: number, - target: AudioBuffer + target: Float32Array ) => { - const channel = target.getChannelData(0); - for (let i = 0; i < channel.length; ++i) { + for (let i = 0; i < target.length; ++i) { // Convert from uint8 to -1..+1 float. - channel[i] = (heap[source + i] / 255) * 2 - 1; + target[i] = (heap[source + i] / 255) * 2 - 1; } return target; }; diff --git a/src/board/index.ts b/src/board/index.ts index fa9f5433..e3752c20 100644 --- a/src/board/index.ts +++ b/src/board/index.ts @@ -1,3 +1,4 @@ +import { create as createResampler } from "@alexanderolsen/libsamplerate-js"; import svgText from "../microbit-drawing.svg"; import { Accelerometer } from "./accelerometer"; import { BoardAudio } from "./audio"; @@ -247,10 +248,22 @@ export class Board { noInitialRun: true, instantiateWasm, }); + + // We update the sample rates before use. + const recordingResampler = await createResampler(1, 48000, 48000); + const defaultResampler = await createResampler(1, 48000, 48000); + const speechResampler = await createResampler(1, 48000, 48000); + // Probably this one is never used so would be nice to avoid + const soundExpressionResampler = await createResampler(1, 48000, 48000); + const module = new ModuleWrapper(wrapped); this.audio.initializeCallbacks({ defaultAudioCallback: wrapped._microbit_hal_audio_raw_ready_callback, + defaultResampler, speechAudioCallback: wrapped._microbit_hal_audio_speech_ready_callback, + speechResampler, + soundExpressionResampler, + recordingResampler, }); this.accelerometer.initializeCallbacks( wrapped._microbit_hal_gesture_callback diff --git a/src/demo.html b/src/demo.html index a887cd86..8f3cd3f9 100644 --- a/src/demo.html +++ b/src/demo.html @@ -99,6 +99,7 @@

MicroPython-micro:bit simulator example embedding

+ diff --git a/src/examples/record.py b/src/examples/record.py index 6c49d0f9..521b2440 100644 --- a/src/examples/record.py +++ b/src/examples/record.py @@ -4,7 +4,8 @@ rate_index = 0 print("Recording...") -my_track = microphone.record(3000) +my_recording = microphone.record(3000) +my_track = my_recording.track() print("Button A to play") while True: if button_a.was_pressed(): diff --git a/src/examples/record_background.py b/src/examples/record_background.py new file mode 100644 index 00000000..2d06859a --- /dev/null +++ b/src/examples/record_background.py @@ -0,0 +1,19 @@ +from microbit import microphone, audio, button_a, button_b, sleep + +rates = [7812, 3906, 15624] +rate_index = 0 + +print("Recording...") +my_recording = audio.AudioRecording(3000) +my_track = microphone.record_into(my_recording, wait=False) +sleep(3000) +print("Button A to play") +while True: + if button_a.was_pressed(): + audio.play(my_track, wait=False) + print("Rate playing", rates[rate_index]) + + if button_b.was_pressed(): + rate_index = (rate_index + 1) % len(rates) + print("Rate change to", rates[rate_index]) + my_track.set_rate(rates[rate_index]) \ No newline at end of file diff --git a/src/jshal.js b/src/jshal.js index c5639732..160cc9c5 100644 --- a/src/jshal.js +++ b/src/jshal.js @@ -230,8 +230,7 @@ mergeInto(LibraryManager.library, { Module.conversions.convertAudioBuffer( Module.HEAPU8, buf, - // @ts-expect-error - Module.board.audio.default.createBuffer(num_samples) + new Float32Array(num_samples) ) ); }, @@ -246,21 +245,10 @@ mergeInto(LibraryManager.library, { /** @type {number} */ num_samples ) { /** @type {AudioBuffer | undefined} */ let webAudioBuffer; - try { - // @ts-expect-error - webAudioBuffer = Module.board.audio.speech.createBuffer(num_samples); - } catch (e) { - // Swallow error on older Safari to keep the sim in a good state. - // @ts-expect-error - if (e.name === "NotSupportedError") { - return; - } else { - throw e; - } - } + const jsBuf = new Float32Array(num_samples); // @ts-expect-error Module.board.audio.speech.writeData( - Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, webAudioBuffer) + Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, jsBuf) ); }, diff --git a/src/mpconfigport.h b/src/mpconfigport.h index 3ae7f209..aa73d9c7 100644 --- a/src/mpconfigport.h +++ b/src/mpconfigport.h @@ -129,6 +129,7 @@ extern uint32_t rng_generate_random_word(void); ((mp_raise_NotImplementedError(MP_ERROR_TEXT("simulator limitation: asm_thumb code"))), p) // The latency of fetching 32 byte audio frames is too much so increase the size -#define AUDIO_OUTPUT_BUFFER_SIZE (64) +// 128 matches the web audio internal buffer sizes +#define AUDIO_OUTPUT_BUFFER_SIZE (128) #endif