From fdf0e347bbf3d7742086e27679c1d316c44d0212 Mon Sep 17 00:00:00 2001 From: Matt Hillsdon Date: Thu, 22 Aug 2024 12:41:51 +0100 Subject: [PATCH] Resample via a libsamplerate We need to do this for recording as we can't specify a rate. We need to do this for playback for sample rates outside of the browser's supported range. That includes the current default rate for Firefox and older Safari. Uses a resampler build that doesn't support medium/best to save on bundle size. I've also increased the buffer size so I can play a 44k sample in an OK-ish way. Maybe 256 will be needed. We know Firefox on Windows still performs poorly. It did badly with audio before these changes. --- README.md | 20 ++ package-lock.json | 475 ++++++++++++++++++++++++++++-- package.json | 3 + src/board/audio/index.ts | 110 ++++--- src/board/conversions.ts | 7 +- src/board/index.ts | 13 + src/demo.html | 1 + src/examples/record.py | 3 +- src/examples/record_background.py | 19 ++ src/jshal.js | 18 +- src/mpconfigport.h | 3 +- 11 files changed, 573 insertions(+), 99 deletions(-) create mode 100644 src/examples/record_background.py diff --git a/README.md b/README.md index 091bccde..40ebdaf0 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,26 @@ Steps for WASM debugging in Chrome: - Enable "WebAssembly Debugging: Enable DWARF support" in DevTools Experiments - DEBUG=1 make +## License + +This software is under the MIT open source license. + +[SPDX-License-Identifier: MIT](LICENSE) + +MicroPython for micro:bit is included in the build process via a submodule. + +We use dependencies via the NPM registry as specified by the package.json file under common Open Source licenses. + +Full details of each package can be found by running `license-checker`: + +```bash +$ npx license-checker --direct --summary --production +``` + +Omit the flags as desired to obtain more detail. + +A fork of libsamplerate_js to reduce bundle size is [hosted on GitHub](https://github.com/microbit-foundation/libsamplerate-js). + ## Code of Conduct Trust, partnership, simplicity and passion are our core values we live and diff --git a/package-lock.json b/package-lock.json index 8fab2c2a..67484c9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "@microbit-foundation/microbit-micropython-v2-simulator", "version": "0.1.0", "license": "MIT", + "dependencies": { + "@alexanderolsen/libsamplerate-js": "microbit-foundation/libsamplerate-js#v2.1.2-microbit.1" + }, "devDependencies": { "@types/emscripten": "^1.39.10", "esbuild": "^0.14.49", @@ -15,6 +18,43 @@ "vitest": "^0.22.1" } }, + "node_modules/@alexanderolsen/libsamplerate-js": { + "version": "2.1.2-microbit.1", + "resolved": "git+ssh://git@github.com/microbit-foundation/libsamplerate-js.git#14bdda3603133d21c07cfb48974617504576a7f7", + "license": "MIT" + }, + "node_modules/@esbuild/android-arm": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.18.tgz", + "integrity": "sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.15.18.tgz", + "integrity": "sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@types/chai": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.3.tgz", @@ -463,9 +503,9 @@ } }, "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "dev": true, "hasInstallScript": true, "optional": true, @@ -483,9 +523,9 @@ "dev": true }, "node_modules/get-func-name": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", + "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", "dev": true, "engines": { "node": "*" @@ -544,10 +584,16 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", - "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -571,15 +617,15 @@ } }, "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==", "dev": true }, "node_modules/postcss": { - "version": "8.4.16", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.16.tgz", - "integrity": "sha512-ipHE1XBvKzm5xI7hiHCZJCSugxvsdq2mPnsq5+UF+VHCjiBvtDrlxJfMBToWaP9D5XlgNmcFGqoHmUn0EYEaRQ==", + "version": "8.4.41", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", + "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==", "dev": true, "funding": [ { @@ -589,12 +635,16 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ], "dependencies": { - "nanoid": "^3.3.4", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" + "nanoid": "^3.3.7", + "picocolors": "^1.0.1", + "source-map-js": "^1.2.0" }, "engines": { "node": "^10 || ^12 || >=14" @@ -633,9 +683,9 @@ } }, "node_modules/rollup": { - "version": "2.77.3", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.77.3.tgz", - "integrity": "sha512-/qxNTG7FbmefJWoeeYJFbHehJ2HNWnjkAFRKzWN/45eNBBF/r8lo992CwcJXEzyVxs5FmfId+vTSTQDb+bxA+g==", + "version": "2.79.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.1.tgz", + "integrity": "sha512-uKxbd0IhMZOhjAiD5oAFp7BqvkA4Dv47qpOCtaNvng4HBwdbWtdOh8f5nZNuk2rp51PMGk3bzfWu5oayNEuYnw==", "dev": true, "bin": { "rollup": "dist/bin/rollup" @@ -648,9 +698,9 @@ } }, "node_modules/source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", "dev": true, "engines": { "node": ">=0.10.0" @@ -696,15 +746,15 @@ } }, "node_modules/vite": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-3.0.9.tgz", - "integrity": "sha512-waYABTM+G6DBTCpYAxvevpG50UOlZuynR0ckTK5PawNVt7ebX6X7wNXHaGIO6wYYFXSM7/WcuFuO2QzhBB6aMw==", + "version": "3.2.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-3.2.10.tgz", + "integrity": "sha512-Dx3olBo/ODNiMVk/cA5Yft9Ws+snLOXrhLtrI3F4XLt4syz2Yg8fayZMWScPKoz12v5BUv7VEmQHnsfpY80fYw==", "dev": true, "dependencies": { - "esbuild": "^0.14.47", - "postcss": "^8.4.16", + "esbuild": "^0.15.9", + "postcss": "^8.4.18", "resolve": "^1.22.1", - "rollup": ">=2.75.6 <2.77.0 || ~2.77.0" + "rollup": "^2.79.1" }, "bin": { "vite": "bin/vite.js" @@ -716,12 +766,17 @@ "fsevents": "~2.3.2" }, "peerDependencies": { + "@types/node": ">= 14", "less": "*", "sass": "*", "stylus": "*", + "sugarss": "*", "terser": "^5.4.0" }, "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, "less": { "optional": true }, @@ -731,11 +786,371 @@ "stylus": { "optional": true }, + "sugarss": { + "optional": true + }, "terser": { "optional": true } } }, + "node_modules/vite/node_modules/esbuild": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.18.tgz", + "integrity": "sha512-x/R72SmW3sSFRm5zrrIjAhCeQSAWoni3CmHEqfQrZIQTM3lVCdehdwuIqaOtfC2slvpdlLa62GYoN8SxT23m6Q==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.15.18", + "@esbuild/linux-loong64": "0.15.18", + "esbuild-android-64": "0.15.18", + "esbuild-android-arm64": "0.15.18", + "esbuild-darwin-64": "0.15.18", + "esbuild-darwin-arm64": "0.15.18", + "esbuild-freebsd-64": "0.15.18", + "esbuild-freebsd-arm64": "0.15.18", + "esbuild-linux-32": "0.15.18", + "esbuild-linux-64": "0.15.18", + "esbuild-linux-arm": "0.15.18", + "esbuild-linux-arm64": "0.15.18", + "esbuild-linux-mips64le": "0.15.18", + "esbuild-linux-ppc64le": "0.15.18", + "esbuild-linux-riscv64": "0.15.18", + "esbuild-linux-s390x": "0.15.18", + "esbuild-netbsd-64": "0.15.18", + "esbuild-openbsd-64": "0.15.18", + "esbuild-sunos-64": "0.15.18", + "esbuild-windows-32": "0.15.18", + "esbuild-windows-64": "0.15.18", + "esbuild-windows-arm64": "0.15.18" + } + }, + "node_modules/vite/node_modules/esbuild-android-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.15.18.tgz", + "integrity": "sha512-wnpt3OXRhcjfIDSZu9bnzT4/TNTDsOUvip0foZOUBG7QbSt//w3QV4FInVJxNhKc/ErhUxc5z4QjHtMi7/TbgA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-android-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.15.18.tgz", + "integrity": "sha512-G4xu89B8FCzav9XU8EjsXacCKSG2FT7wW9J6hOc18soEHJdtWu03L3TQDGf0geNxfLTtxENKBzMSq9LlbjS8OQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-darwin-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.18.tgz", + "integrity": "sha512-2WAvs95uPnVJPuYKP0Eqx+Dl/jaYseZEUUT1sjg97TJa4oBtbAKnPnl3b5M9l51/nbx7+QAEtuummJZW0sBEmg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-darwin-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.15.18.tgz", + "integrity": "sha512-tKPSxcTJ5OmNb1btVikATJ8NftlyNlc8BVNtyT/UAr62JFOhwHlnoPrhYWz09akBLHI9nElFVfWSTSRsrZiDUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-freebsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.15.18.tgz", + "integrity": "sha512-TT3uBUxkteAjR1QbsmvSsjpKjOX6UkCstr8nMr+q7zi3NuZ1oIpa8U41Y8I8dJH2fJgdC3Dj3CXO5biLQpfdZA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-freebsd-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.15.18.tgz", + "integrity": "sha512-R/oVr+X3Tkh+S0+tL41wRMbdWtpWB8hEAMsOXDumSSa6qJR89U0S/PpLXrGF7Wk/JykfpWNokERUpCeHDl47wA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-32": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.15.18.tgz", + "integrity": "sha512-lphF3HiCSYtaa9p1DtXndiQEeQDKPl9eN/XNoBf2amEghugNuqXNZA/ZovthNE2aa4EN43WroO0B85xVSjYkbg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.15.18.tgz", + "integrity": "sha512-hNSeP97IviD7oxLKFuii5sDPJ+QHeiFTFLoLm7NZQligur8poNOWGIgpQ7Qf8Balb69hptMZzyOBIPtY09GZYw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-arm": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.15.18.tgz", + "integrity": "sha512-UH779gstRblS4aoS2qpMl3wjg7U0j+ygu3GjIeTonCcN79ZvpPee12Qun3vcdxX+37O5LFxz39XeW2I9bybMVA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.15.18.tgz", + "integrity": "sha512-54qr8kg/6ilcxd+0V3h9rjT4qmjc0CccMVWrjOEM/pEcUzt8X62HfBSeZfT2ECpM7104mk4yfQXkosY8Quptug==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-mips64le": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.15.18.tgz", + "integrity": "sha512-Mk6Ppwzzz3YbMl/ZZL2P0q1tnYqh/trYZ1VfNP47C31yT0K8t9s7Z077QrDA/guU60tGNp2GOwCQnp+DYv7bxQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-ppc64le": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.15.18.tgz", + "integrity": "sha512-b0XkN4pL9WUulPTa/VKHx2wLCgvIAbgwABGnKMY19WhKZPT+8BxhZdqz6EgkqCLld7X5qiCY2F/bfpUUlnFZ9w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-riscv64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.15.18.tgz", + "integrity": "sha512-ba2COaoF5wL6VLZWn04k+ACZjZ6NYniMSQStodFKH/Pu6RxzQqzsmjR1t9QC89VYJxBeyVPTaHuBMCejl3O/xg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-linux-s390x": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.15.18.tgz", + "integrity": "sha512-VbpGuXEl5FCs1wDVp93O8UIzl3ZrglgnSQ+Hu79g7hZu6te6/YHgVJxCM2SqfIila0J3k0csfnf8VD2W7u2kzQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-netbsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.15.18.tgz", + "integrity": "sha512-98ukeCdvdX7wr1vUYQzKo4kQ0N2p27H7I11maINv73fVEXt2kyh4K4m9f35U1K43Xc2QGXlzAw0K9yoU7JUjOg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-openbsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.15.18.tgz", + "integrity": "sha512-yK5NCcH31Uae076AyQAXeJzt/vxIo9+omZRKj1pauhk3ITuADzuOx5N2fdHrAKPxN+zH3w96uFKlY7yIn490xQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-sunos-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.18.tgz", + "integrity": "sha512-On22LLFlBeLNj/YF3FT+cXcyKPEI263nflYlAhz5crxtp3yRG1Ugfr7ITyxmCmjm4vbN/dGrb/B7w7U8yJR9yw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-windows-32": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.15.18.tgz", + "integrity": "sha512-o+eyLu2MjVny/nt+E0uPnBxYuJHBvho8vWsC2lV61A7wwTWC3jkN2w36jtA+yv1UgYkHRihPuQsL23hsCYGcOQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-windows-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.15.18.tgz", + "integrity": "sha512-qinug1iTTaIIrCorAUjR0fcBk24fjzEedFYhhispP8Oc7SFvs+XeW3YpAKiKp8dRpizl4YYAhxMjlftAMJiaUw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/esbuild-windows-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.15.18.tgz", + "integrity": "sha512-q9bsYzegpZcLziq0zgUi5KqGVtfhjxGbnksaBFYmWLxeV/S1fK4OLdq2DFYnXcLMjlZw2L0jLsk1eGoB522WXQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/vitest": { "version": "0.22.1", "resolved": "https://registry.npmjs.org/vitest/-/vitest-0.22.1.tgz", diff --git a/package.json b/package.json index 5856f89c..2bee2d66 100644 --- a/package.json +++ b/package.json @@ -28,5 +28,8 @@ "esbuild": "^0.14.49", "prettier": "2.6.0", "vitest": "^0.22.1" + }, + "dependencies": { + "@alexanderolsen/libsamplerate-js": "microbit-foundation/libsamplerate-js#v2.1.2-microbit.1" } } diff --git a/src/board/audio/index.ts b/src/board/audio/index.ts index 9dd127bf..1fdb67e2 100644 --- a/src/board/audio/index.ts +++ b/src/board/audio/index.ts @@ -1,6 +1,11 @@ +import { SRC } from "@alexanderolsen/libsamplerate-js/dist/src"; import { replaceBuiltinSound } from "./built-in-sounds"; import { SoundEmojiSynthesizer } from "./sound-emoji-synthesizer"; import { parseSoundEffects } from "./sound-expressions"; +import { + create as createSampleRateConverter, + ConverterType, +} from "@alexanderolsen/libsamplerate-js"; declare global { interface Window { @@ -11,7 +16,11 @@ declare global { interface AudioOptions { defaultAudioCallback: () => void; + defaultResampler: SRC; speechAudioCallback: () => void; + speechResampler: SRC; + soundExpressionResampler: SRC; + recordingResampler: SRC; } export class BoardAudio { @@ -24,6 +33,8 @@ export class BoardAudio { private muteNode: GainNode | undefined; private sensitivityNode: GainNode | undefined; + private recordingResampler: SRC | undefined; + default: BufferedAudio | undefined; speech: BufferedAudio | undefined; soundExpression: BufferedAudio | undefined; @@ -34,11 +45,17 @@ export class BoardAudio { initializeCallbacks({ defaultAudioCallback, + defaultResampler, speechAudioCallback, + speechResampler, + soundExpressionResampler, + recordingResampler, }: AudioOptions) { if (!this.context) { throw new Error("Context must be pre-created from a user event"); } + this.recordingResampler = recordingResampler; + this.muteNode = this.context.createGain(); this.muteNode.gain.setValueAtTime( this.muted ? 0 : 1, @@ -56,16 +73,19 @@ export class BoardAudio { this.default = new BufferedAudio( this.context, this.volumeNode, + defaultResampler, defaultAudioCallback ); this.speech = new BufferedAudio( this.context, this.volumeNode, + speechResampler, speechAudioCallback ); this.soundExpression = new BufferedAudio( this.context, this.volumeNode, + soundExpressionResampler, () => { if (this.currentSoundExpressionCallback) { this.currentSoundExpressionCallback(); @@ -75,12 +95,13 @@ export class BoardAudio { } async createAudioContextFromUserInteraction(): Promise { + // If we set a 44.1kHz rate then we fail to connect to user media on Mac as it selects 48000 + // So we leave it at the default hoping it's most likely to match user media... + // Until there's progress on this there doesn't seem a better way: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1674892 this.context = - this.context ?? - new (window.AudioContext || window.webkitAudioContext)({ - // The highest rate is the sound expression synth. - sampleRate: 44100, - }); + this.context ?? new (window.AudioContext || window.webkitAudioContext)(); + if (this.context.state === "suspended") { return this.context.resume(); } @@ -92,21 +113,16 @@ export class BoardAudio { this.stopSoundExpression(); }; const synth = new SoundEmojiSynthesizer(0, onDone); + this.soundExpression!.setSampleRate(synth.sampleRate); synth.play(soundEffects); const callback = () => { const source = synth.pull(); if (this.context) { - // Use createBuffer instead of new AudioBuffer to support Safari 14.0. - const target = this.context.createBuffer( - 1, - source.length, - synth.sampleRate - ); - const channel = target.getChannelData(0); + const target = new Float32Array(source.length); for (let i = 0; i < source.length; i++) { // Buffer is (0, 1023) we need to map it to (-1, 1) - channel[i] = (source[i] - 512) / 512; + target[i] = (source[i] - 512) / 512; } this.soundExpression!.writeData(target); } @@ -197,6 +213,7 @@ export class BoardAudio { try { micStream = await navigator.mediaDevices.getUserMedia({ video: false, + // It seems Firefox ignores the rate set here audio: true, }); } catch (e) { @@ -208,41 +225,34 @@ export class BoardAudio { const source = this.context!.createMediaStreamSource(micStream); source.connect(this.sensitivityNode!); - // TODO: consider AudioWorklet - worth it? Browser support? - // consider alternative resampling approaches - // what sample rates are actually supported this way? + const recorder = this.context!.createScriptProcessor(2048, 1, 1); + + const inputSampleRate = this.context!.sampleRate; + this.recordingResampler!.inputSampleRate = inputSampleRate; + this.recordingResampler!.outputSampleRate = sampleRate; + recorder.onaudioprocess = (e) => { - const offlineContext = new (window.OfflineAudioContext || - window.webkitOfflineAudioContext)( - 1, - sampleRate * (e.inputBuffer.length / e.inputBuffer.sampleRate), - sampleRate + const resampled = this.recordingResampler!.full( + e.inputBuffer.getChannelData(0) ); - const source = offlineContext.createBufferSource(); - source.buffer = e.inputBuffer; - source.connect(offlineContext.destination); - source.start(); - offlineContext.addEventListener("complete", (e) => { - onChunk(e.renderedBuffer.getChannelData(0)); - samplesSent += e.renderedBuffer.length; - if (samplesSent >= samplesNeeded) { - this.stopRecording(); - } - }); - offlineContext.startRendering(); + onChunk(resampled); + samplesSent += resampled.length; + if (samplesSent >= samplesNeeded) { + this.stopRecording(); + } }; - this.sensitivityNode!.connect(recorder); - recorder.connect(this.context!.destination); - this.stopActiveRecording = () => { recorder.disconnect(); this.sensitivityNode!.disconnect(); source.disconnect(); - micStream.getTracks().forEach((track) => track.stop()); + micStream?.getTracks().forEach((track) => track.stop()); this.microphoneEl.style.display = "none"; this.stopActiveRecording = undefined; }; + + this.sensitivityNode!.connect(recorder); + recorder.connect(this.context!.destination); } boardStopped() { @@ -263,31 +273,35 @@ export class BoardAudio { class BufferedAudio { nextStartTime: number = -1; - private sampleRate: number = -1; constructor( private context: AudioContext, private destination: AudioNode, + private resampler: SRC, private callback: () => void - ) {} + ) { + this.resampler.outputSampleRate = this.context.sampleRate; + } init(sampleRate: number) { // This is called for each new audio source so don't reset nextStartTime // or we start to overlap audio - this.sampleRate = sampleRate; - } - - createBuffer(length: number) { - // Use createBuffer instead of new AudioBuffer to support Safari 14.0. - return this.context.createBuffer(1, length, this.sampleRate); + this.setSampleRate(sampleRate); } setSampleRate(sampleRate: number) { - this.sampleRate = sampleRate; + this.resampler.inputSampleRate = sampleRate; } - writeData(buffer: AudioBuffer) { - // Use createBufferSource instead of new AudioBufferSourceNode to support Safari 14.0. + writeData(data: Float32Array) { + // In practice the supported range is less than the 8k..96k required by the spec and varies by browser + // for a consistent performance profile we're always resampling for now rather than letting Web Audio do it + let sampleRate = this.context.sampleRate; + data = this.resampler.full(data); + + // Use createXXX instead to support Safari 14.0. + const buffer = this.context.createBuffer(1, data.length, sampleRate); + buffer.copyToChannel(data, 0); const source = this.context.createBufferSource(); source.buffer = buffer; source.onended = this.callCallback; diff --git a/src/board/conversions.ts b/src/board/conversions.ts index 392d7808..f42bca7f 100644 --- a/src/board/conversions.ts +++ b/src/board/conversions.ts @@ -98,12 +98,11 @@ export function convertAccelerometerNumberToString(value: number): string { export const convertAudioBuffer = ( heap: Uint8Array, source: number, - target: AudioBuffer + target: Float32Array ) => { - const channel = target.getChannelData(0); - for (let i = 0; i < channel.length; ++i) { + for (let i = 0; i < target.length; ++i) { // Convert from uint8 to -1..+1 float. - channel[i] = (heap[source + i] / 255) * 2 - 1; + target[i] = (heap[source + i] / 255) * 2 - 1; } return target; }; diff --git a/src/board/index.ts b/src/board/index.ts index fa9f5433..e3752c20 100644 --- a/src/board/index.ts +++ b/src/board/index.ts @@ -1,3 +1,4 @@ +import { create as createResampler } from "@alexanderolsen/libsamplerate-js"; import svgText from "../microbit-drawing.svg"; import { Accelerometer } from "./accelerometer"; import { BoardAudio } from "./audio"; @@ -247,10 +248,22 @@ export class Board { noInitialRun: true, instantiateWasm, }); + + // We update the sample rates before use. + const recordingResampler = await createResampler(1, 48000, 48000); + const defaultResampler = await createResampler(1, 48000, 48000); + const speechResampler = await createResampler(1, 48000, 48000); + // Probably this one is never used so would be nice to avoid + const soundExpressionResampler = await createResampler(1, 48000, 48000); + const module = new ModuleWrapper(wrapped); this.audio.initializeCallbacks({ defaultAudioCallback: wrapped._microbit_hal_audio_raw_ready_callback, + defaultResampler, speechAudioCallback: wrapped._microbit_hal_audio_speech_ready_callback, + speechResampler, + soundExpressionResampler, + recordingResampler, }); this.accelerometer.initializeCallbacks( wrapped._microbit_hal_gesture_callback diff --git a/src/demo.html b/src/demo.html index a887cd86..8f3cd3f9 100644 --- a/src/demo.html +++ b/src/demo.html @@ -99,6 +99,7 @@

MicroPython-micro:bit simulator example embedding

+ diff --git a/src/examples/record.py b/src/examples/record.py index 6c49d0f9..521b2440 100644 --- a/src/examples/record.py +++ b/src/examples/record.py @@ -4,7 +4,8 @@ rate_index = 0 print("Recording...") -my_track = microphone.record(3000) +my_recording = microphone.record(3000) +my_track = my_recording.track() print("Button A to play") while True: if button_a.was_pressed(): diff --git a/src/examples/record_background.py b/src/examples/record_background.py new file mode 100644 index 00000000..2d06859a --- /dev/null +++ b/src/examples/record_background.py @@ -0,0 +1,19 @@ +from microbit import microphone, audio, button_a, button_b, sleep + +rates = [7812, 3906, 15624] +rate_index = 0 + +print("Recording...") +my_recording = audio.AudioRecording(3000) +my_track = microphone.record_into(my_recording, wait=False) +sleep(3000) +print("Button A to play") +while True: + if button_a.was_pressed(): + audio.play(my_track, wait=False) + print("Rate playing", rates[rate_index]) + + if button_b.was_pressed(): + rate_index = (rate_index + 1) % len(rates) + print("Rate change to", rates[rate_index]) + my_track.set_rate(rates[rate_index]) \ No newline at end of file diff --git a/src/jshal.js b/src/jshal.js index c5639732..160cc9c5 100644 --- a/src/jshal.js +++ b/src/jshal.js @@ -230,8 +230,7 @@ mergeInto(LibraryManager.library, { Module.conversions.convertAudioBuffer( Module.HEAPU8, buf, - // @ts-expect-error - Module.board.audio.default.createBuffer(num_samples) + new Float32Array(num_samples) ) ); }, @@ -246,21 +245,10 @@ mergeInto(LibraryManager.library, { /** @type {number} */ num_samples ) { /** @type {AudioBuffer | undefined} */ let webAudioBuffer; - try { - // @ts-expect-error - webAudioBuffer = Module.board.audio.speech.createBuffer(num_samples); - } catch (e) { - // Swallow error on older Safari to keep the sim in a good state. - // @ts-expect-error - if (e.name === "NotSupportedError") { - return; - } else { - throw e; - } - } + const jsBuf = new Float32Array(num_samples); // @ts-expect-error Module.board.audio.speech.writeData( - Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, webAudioBuffer) + Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, jsBuf) ); }, diff --git a/src/mpconfigport.h b/src/mpconfigport.h index 3ae7f209..aa73d9c7 100644 --- a/src/mpconfigport.h +++ b/src/mpconfigport.h @@ -129,6 +129,7 @@ extern uint32_t rng_generate_random_word(void); ((mp_raise_NotImplementedError(MP_ERROR_TEXT("simulator limitation: asm_thumb code"))), p) // The latency of fetching 32 byte audio frames is too much so increase the size -#define AUDIO_OUTPUT_BUFFER_SIZE (64) +// 128 matches the web audio internal buffer sizes +#define AUDIO_OUTPUT_BUFFER_SIZE (128) #endif