From 229f167c9e08237e548ece7c6c750b260631208b Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 23 May 2023 17:31:04 +0200 Subject: [PATCH 01/37] Send bytes in chunks --- .../CameraCaptureRawBytes.ino | 84 ++++++++++++++----- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino index 65458f26a..a7146e60e 100644 --- a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino +++ b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino @@ -6,8 +6,10 @@ Camera cam(galaxyCore); #define IMAGE_MODE CAMERA_RGB565 #elif defined(ARDUINO_PORTENTA_H7_M7) - #include "hm0360.h" - HM0360 himax; + //#include "hm0360.h" + //HM0360 himax; + #include "himax.h"; + HM01B0 himax; Camera cam(himax); #define IMAGE_MODE CAMERA_GRAYSCALE #elif defined(ARDUINO_GIGA) @@ -33,6 +35,8 @@ If resolution higher than 320x240 is required, please use external RAM via // and adding in setup() SDRAM.begin(); */ +#define CHUNK_SIZE 512 // Size of chunks in bytes +#define RESOLUTION CAMERA_R320x240 FrameBuffer fb; unsigned long lastUpdate = 0; @@ -40,7 +44,8 @@ unsigned long lastUpdate = 0; void blinkLED(uint32_t count = 0xFFFFFFFF) { - pinMode(LED_BUILTIN, OUTPUT); + pinMode(LED_BUILTIN, OUTPUT); + while (count--) { digitalWrite(LED_BUILTIN, LOW); // turn the LED on (HIGH is the voltage level) delay(50); // wait for a second @@ -49,38 +54,71 @@ void blinkLED(uint32_t count = 0xFFFFFFFF) } } +void clearSerialBuffer(){ + while(Serial.available()){ + Serial.read(); + } + Serial.end(); + Serial.begin(9600); +} + void setup() { // Init the cam QVGA, 30FPS - if (!cam.begin(CAMERA_R320x240, IMAGE_MODE, 30)) { + if (!cam.begin(RESOLUTION, IMAGE_MODE, 30)) { blinkLED(); } blinkLED(5); + + pinMode(LEDB, OUTPUT); + digitalWrite(LEDB, HIGH); +} + +void sendFrame(){ + // Grab frame and write to serial + if (cam.grabFrame(fb, 3000) == 0) { + byte* buffer = fb.getBuffer(); + size_t bufferSize = cam.frameSize(); + digitalWrite(LEDB, LOW); + + // Split buffer into chunks + for(size_t i = 0; i < bufferSize; i += CHUNK_SIZE) { + size_t chunkSize = min(bufferSize - i, CHUNK_SIZE); + Serial.write(buffer + i, chunkSize); + Serial.flush(); + delay(1); // Optional: Add a small delay to allow the receiver to process the chunk + } + + digitalWrite(LEDB, HIGH); + } else { + blinkLED(20); + } +} + +void sendCameraConfig(){ + Serial.write(RESOLUTION); + Serial.write(IMAGE_MODE); + Serial.flush(); + delay(1); } void loop() { if(!Serial) { - Serial.begin(921600); - while(!Serial); + Serial.begin(115200, SERIAL_8E2); + while(!Serial); } - // Time out after 2 seconds, which sets the (constant) frame rate - bool timeoutDetected = millis() - lastUpdate > 2000; - - // Wait for sync byte and timeout - // Notice that this order must be kept, or the sync bytes will be - // consumed prematurely - if ((!timeoutDetected) || (Serial.read() != 1)) - { - return; - } + if(!Serial.available()) return; - lastUpdate = millis(); - - // Grab frame and write to serial - if (cam.grabFrame(fb, 3000) == 0) { - Serial.write(fb.getBuffer(), cam.frameSize()); - } else { - blinkLED(20); + byte request = Serial.read(); + + switch(request){ + case 1: + sendFrame(); + break; + case 2: + sendCameraConfig(); + break; } + } From ea9c54525bb46b91b430f40addb3267bbebcd3f8 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 23 May 2023 17:31:20 +0200 Subject: [PATCH 02/37] Add web serial camera app --- .../Camera/extras/WebSerialCamera/app.js | 246 ++++++++++++++++++ .../extras/WebSerialCamera/conversion.js | 11 + .../Camera/extras/WebSerialCamera/index.html | 28 ++ 3 files changed, 285 insertions(+) create mode 100644 libraries/Camera/extras/WebSerialCamera/app.js create mode 100644 libraries/Camera/extras/WebSerialCamera/conversion.js create mode 100644 libraries/Camera/extras/WebSerialCamera/index.html diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js new file mode 100644 index 000000000..64e0beb60 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -0,0 +1,246 @@ +const connectButton = document.getElementById('connect'); +const refreshButton = document.getElementById('refresh'); +const startButton = document.getElementById('start'); +const disconnectButton = document.getElementById('disconnect'); +const canvas = document.getElementById('bitmapCanvas'); +const ctx = canvas.getContext('2d'); + +const UserActionAbortError = 8; +const ArduinoUSBVendorId = 0x2341; + +const imageWidth = 320; // Adjust this value based on your bitmap width +const imageHeight = 240; // Adjust this value based on your bitmap height +const bytesPerPixel = 1; // Adjust this value based on your bitmap format +// const mode = 'RGB565'; // Adjust this value based on your bitmap format +const totalBytes = imageWidth * imageHeight * bytesPerPixel; + +// Set the buffer size to the total bytes. This allows to read the entire bitmap in one go. +const bufferSize = Math.min(totalBytes, 16 * 1024 * 1024); // Max buffer size is 16MB +const baudRate = 115200; // Adjust this value based on your device's baud rate +const dataBits = 8; // Adjust this value based on your device's data bits +const stopBits = 2; // Adjust this value based on your device's stop bits + +let currentPort, currentReader; + +async function requestSerialPort(){ + try { + // Request a serial port + const port = await navigator.serial.requestPort({ filters: [{ usbVendorId: ArduinoUSBVendorId }] }); + currentPort = port; + return port; + } catch (error) { + if(error.code != UserActionAbortError){ + console.log(error); + } + return null; + } +} + +async function autoConnect(){ + if(currentPort){ + console.log('🔌 Already connected to a serial port.'); + return false; + } + + // Get all serial ports the user has previously granted the website access to. + const ports = await navigator.serial.getPorts(); + + for (const port of ports) { + console.log('👀 Serial port found with VID: 0x' + port.getInfo().usbVendorId.toString(16)); + if(port.getInfo().usbVendorId === ArduinoUSBVendorId){ + currentPort = port; + return await connectSerial(currentPort); + } + } + return false; +} + +async function connectSerial(port, baudRate = 115200, dataBits = 8, stopBits = 2, bufferSize = 4096) { + try { + // If the port is already open, close it + if (port.readable) await port.close(); + await port.open({ baudRate: baudRate, parity: "even", dataBits: dataBits, stopBits: stopBits, bufferSize: bufferSize }); + console.log('✅ Connected to serial port.'); + return true; + } catch (error) { + return false; + } +} + + +async function readBytes(port, numBytes, timeout = null){ + if(port.readable.locked){ + console.log('🔒 Stream is already locked. Ignoring request...'); + return null; + } + + const bytesRead = new Uint8Array(numBytes); + let bytesReadIdx = 0; + let keepReading = true; + + // As long as the errors are non-fatal, a new ReadableStream is created automatically and hence port.readable is non-null. + // If a fatal error occurs, such as the serial device being removed, then port.readable becomes null. + + while (port.readable && keepReading) { + const reader = port.readable.getReader(); + currentReader = reader; + let timeoutID = null; + let count = 0; + + try { + while (bytesReadIdx < numBytes) { + if(timeout){ + timeoutID = setTimeout(() => { + console.log('⌛️ Timeout occurred while reading.'); + if(port.readable) reader?.cancel(); + }, timeout); + } + + const { value, done } = await reader.read(); + if(timeoutID) clearTimeout(timeoutID); + + if(value){ + for (const byte of value) { + bytesRead[bytesReadIdx++] = byte; + } + // count += value.byteLength; + // console.log(`Read ${value.byteLength} (Total: ${count}) out of ${numBytes} bytes.}`); + } + + if (done) { + // |reader| has been canceled. + console.log('🚫 Reader has been canceled'); + break; + } + } + + } catch (error) { + // Handle |error|... + console.log('💣 Error occurred while reading: '); + console.log(error); + } finally { + keepReading = false; + // console.log('🔓 Releasing reader lock...'); + reader?.releaseLock(); + currentReader = null; + } + } + return bytesRead; +} + +function renderBitmap(bytes, width, height) { + canvas.width = width; + canvas.height = height; + const BYTES_PER_ROW = width * bytesPerPixel; + const BYTES_PER_COL = height * bytesPerPixel; + + const imageData = ctx.createImageData(canvas.width, canvas.height); + const data = imageData.data; + + for (let row = 0; row < BYTES_PER_ROW; row++) { + for (let col = 0; col < BYTES_PER_COL; col++) { + const byte = bytes[row * BYTES_PER_COL + col]; + const grayscaleValue = byte; + + const idx = (row * BYTES_PER_COL + col) * 4; + data[idx] = grayscaleValue; // Red channel + data[idx + 1] = grayscaleValue; // Green channel + data[idx + 2] = grayscaleValue; // Blue channel + data[idx + 3] = 255; // Alpha channel (opacity) + } + } + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.putImageData(imageData, 0, 0); +} + +async function requestFrame(port){ + if(!port?.writable) { + console.log('🚫 Port is not writable. Ignoring request...'); + return; + } + // console.log('Writing 1 to the serial port...'); + // Write a 1 to the serial port + const writer = port.writable.getWriter(); + await writer.write(new Uint8Array([1])); + await writer.close(); +} +async function renderStream(){ + while(true && currentPort){ + await renderFrame(currentPort); + } +} + +async function renderFrame(port){ + if(!port) return; + const bytes = await getFrame(port); + if(!bytes) return false; // Nothing to render + // console.log(`Reading done ✅. Rendering image...`); + // Render the bytes as a grayscale bitmap + renderBitmap(bytes, imageWidth, imageHeight); + return true; +} + +async function getFrame(port) { + if(!port) return; + + await requestFrame(port); + // console.log(`Trying to read ${totalBytes} bytes...`); + // Read the given amount of bytes + return await readBytes(port, totalBytes, 2000); +} + +async function disconnectSerial(port) { + if(!port) return; + try { + currentPort = null; + await currentReader?.cancel(); + await port.close(); + console.log('🔌 Disconnected from serial port.'); + } catch (error) { + console.error('💣 Error occurred while disconnecting: ' + error.message); + }; +} + +startButton.addEventListener('click', renderStream); +connectButton.addEventListener('click', async () => { + currentPort = await requestSerialPort(); + if(await connectSerial(currentPort, baudRate, dataBits, stopBits, bufferSize)){ + renderStream(); + } +}); +disconnectButton.addEventListener('click', () => disconnectSerial(currentPort)); +refreshButton.addEventListener('click', () => { + renderFrame(currentPort); +}); + +navigator.serial.addEventListener("connect", (e) => { + // Connect to `e.target` or add it to a list of available ports. + console.log('🔌 Serial port became available. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); + autoConnect().then((connected) => { + if(connected){ + renderStream(); + }; + }); +}); + +navigator.serial.addEventListener("disconnect", (e) => { + // Remove `e.target` from the list of available ports. + console.log('❌ Serial port lost. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); + currentPort = null; +}); + +// On page load event, try to connect to the serial port +window.addEventListener('load', async () => { + console.log('🚀 Page loaded. Trying to connect to serial port...'); + setTimeout(() => { + autoConnect().then((connected) => { + if (connected) { + renderStream(); + }; + }); + }, 1000); +}); + +if (!("serial" in navigator)) { + alert("The Web Serial API is not supported in your browser."); +} \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/conversion.js b/libraries/Camera/extras/WebSerialCamera/conversion.js new file mode 100644 index 000000000..6718b38e0 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/conversion.js @@ -0,0 +1,11 @@ +function convertRGB565ToRGB888(pixelValue) { + // RGB565 + let r = (pixelValue >> (6 + 5)) & 0x1F; + let g = (pixelValue >> 5) & 0x3F; + let b = pixelValue & 0x1F; + // RGB888 - amplify + r <<= 3; + g <<= 2; + b <<= 3; + return [r, g, b]; +} diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html new file mode 100644 index 000000000..29c07856c --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -0,0 +1,28 @@ + + + + + + Web Serial Bitmap Reader + + + +
+ +
+ + + + +
+
+ + + From 940cf827d99f7fd6903e50bcbbde4b0aee83f379 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 23 May 2023 17:46:26 +0200 Subject: [PATCH 03/37] Catch edge case --- libraries/Camera/extras/WebSerialCamera/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 64e0beb60..4913b82e9 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -102,6 +102,7 @@ async function readBytes(port, numBytes, timeout = null){ if(value){ for (const byte of value) { bytesRead[bytesReadIdx++] = byte; + if (bytesReadIdx >= numBytes) break; } // count += value.byteLength; // console.log(`Read ${value.byteLength} (Total: ${count}) out of ${numBytes} bytes.}`); From 70f6b23923975c61390eeab7f7f7f09ef035ea26 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 10:23:14 +0200 Subject: [PATCH 04/37] Remove unused code --- .../CameraCaptureRawBytes/CameraCaptureRawBytes.ino | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino index a7146e60e..c4c88ec63 100644 --- a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino +++ b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino @@ -41,7 +41,6 @@ FrameBuffer fb; unsigned long lastUpdate = 0; - void blinkLED(uint32_t count = 0xFFFFFFFF) { pinMode(LED_BUILTIN, OUTPUT); @@ -54,14 +53,6 @@ void blinkLED(uint32_t count = 0xFFFFFFFF) } } -void clearSerialBuffer(){ - while(Serial.available()){ - Serial.read(); - } - Serial.end(); - Serial.begin(9600); -} - void setup() { // Init the cam QVGA, 30FPS if (!cam.begin(RESOLUTION, IMAGE_MODE, 30)) { From d54dba8d308cadfe60a9f212f79fd1de4b8909a8 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 10:24:05 +0200 Subject: [PATCH 05/37] Working version for RGB --- .../Camera/extras/WebSerialCamera/app.js | 57 +++++++++++++--- .../extras/WebSerialCamera/conversion.js | 8 +++ .../Camera/extras/WebSerialCamera/index.html | 10 +-- .../Camera/extras/WebSerialCamera/style.css | 67 +++++++++++++++++++ 4 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 libraries/Camera/extras/WebSerialCamera/style.css diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 4913b82e9..01ce02425 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -8,14 +8,31 @@ const ctx = canvas.getContext('2d'); const UserActionAbortError = 8; const ArduinoUSBVendorId = 0x2341; +let config = { + "RGB565": { + "convert": convertRGB565ToRGB888, + "bytesPerPixel": 2 + }, + "GRAYSCALE": { + "convert": convertGrayScaleToRGB888, + "bytesPerPixel": 1 + }, + "RGB888": { + "convert": convertToRGB888, + "bytesPerPixel": 3 + } +}; + const imageWidth = 320; // Adjust this value based on your bitmap width const imageHeight = 240; // Adjust this value based on your bitmap height -const bytesPerPixel = 1; // Adjust this value based on your bitmap format +const bytesPerPixel = 2; // Adjust this value based on your bitmap format // const mode = 'RGB565'; // Adjust this value based on your bitmap format -const totalBytes = imageWidth * imageHeight * bytesPerPixel; +const mode = 'GRAYSCALE'; // Adjust this value based on your bitmap format +const totalBytes = imageWidth * imageHeight * config[mode].bytesPerPixel; // Set the buffer size to the total bytes. This allows to read the entire bitmap in one go. const bufferSize = Math.min(totalBytes, 16 * 1024 * 1024); // Max buffer size is 16MB +const flowControl = 'hardware'; const baudRate = 115200; // Adjust this value based on your device's baud rate const dataBits = 8; // Adjust this value based on your device's data bits const stopBits = 2; // Adjust this value based on your device's stop bits @@ -85,7 +102,7 @@ async function readBytes(port, numBytes, timeout = null){ const reader = port.readable.getReader(); currentReader = reader; let timeoutID = null; - let count = 0; + // let count = 0; try { while (bytesReadIdx < numBytes) { @@ -129,9 +146,26 @@ async function readBytes(port, numBytes, timeout = null){ return bytesRead; } +// Get the pixel value using big endian +// Big-endian: the most significant byte comes first +function getPixelValue(data, index, bytesPerPixel){ + if(bytesPerPixel == 1){ + return data[index]; + } else if(bytesPerPixel == 2){ + return (data[index] << 8) | data[index + 1]; + } else if(bytesPerPixel == 3){ + return (data[index] << 16) | (data[index + 1] << 8) | data[index + 2]; + } else if(bytesPerPixel == 4){ + return (data[index] << 24) | (data[index + 1] << 16) | (data[index + 2] << 8) | data[index + 3]; + } + + return 0; +} + function renderBitmap(bytes, width, height) { canvas.width = width; canvas.height = height; + const bytesPerPixel = config[mode].bytesPerPixel; const BYTES_PER_ROW = width * bytesPerPixel; const BYTES_PER_COL = height * bytesPerPixel; @@ -140,13 +174,14 @@ function renderBitmap(bytes, width, height) { for (let row = 0; row < BYTES_PER_ROW; row++) { for (let col = 0; col < BYTES_PER_COL; col++) { - const byte = bytes[row * BYTES_PER_COL + col]; - const grayscaleValue = byte; - - const idx = (row * BYTES_PER_COL + col) * 4; - data[idx] = grayscaleValue; // Red channel - data[idx + 1] = grayscaleValue; // Green channel - data[idx + 2] = grayscaleValue; // Blue channel + const dataIndex = (row * BYTES_PER_ROW) + (col * bytesPerPixel); + const pixelValue = getPixelValue(bytes, dataIndex, bytesPerPixel); + const [r, g, b] = config[mode].convert(pixelValue); + + const idx = ((row * width) + col) * 4; + data[idx] = r; // Red channel + data[idx + 1] = g; // Green channel + data[idx + 2] = b; // Blue channel data[idx + 3] = 255; // Alpha channel (opacity) } } @@ -205,7 +240,7 @@ async function disconnectSerial(port) { startButton.addEventListener('click', renderStream); connectButton.addEventListener('click', async () => { currentPort = await requestSerialPort(); - if(await connectSerial(currentPort, baudRate, dataBits, stopBits, bufferSize)){ + if(await connectSerial(currentPort, baudRate, dataBits, stopBits, bufferSize, flowControl)){ renderStream(); } }); diff --git a/libraries/Camera/extras/WebSerialCamera/conversion.js b/libraries/Camera/extras/WebSerialCamera/conversion.js index 6718b38e0..ad467c077 100644 --- a/libraries/Camera/extras/WebSerialCamera/conversion.js +++ b/libraries/Camera/extras/WebSerialCamera/conversion.js @@ -9,3 +9,11 @@ function convertRGB565ToRGB888(pixelValue) { b <<= 3; return [r, g, b]; } + +function convertGrayScaleToRGB888(pixelValue) { + return [pixelValue, pixelValue, pixelValue]; +} + +function convertToRGB888(pixelValue){ + return [pixelValue[0], pixelValue[1], pixelValue[2]]; +} \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index 29c07856c..408aeac66 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -4,14 +4,7 @@ Web Serial Bitmap Reader - +
@@ -23,6 +16,7 @@
+ diff --git a/libraries/Camera/extras/WebSerialCamera/style.css b/libraries/Camera/extras/WebSerialCamera/style.css new file mode 100644 index 000000000..ef9f7832f --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/style.css @@ -0,0 +1,67 @@ +:root { + --main-control-color: #008184; + --main-control-color-hover: #005c5f; + --main-flexbox-gap: 16px; + --secondary-text-color: #87898b; +} + +html { + font-size: 14px; +} + +body { + font-family: 'Open Sans', sans-serif; + text-align: center; +} + +#main-container { + display: flex; + flex-direction: column; + align-items: center; + gap: 1rem; + margin-top: 20px; +} + +#controls { + display: flex; + flex-direction: row; + align-items: center; + gap: 1rem; + margin-top: 20px; +} + +canvas { + border-radius: 5px; +} + +button { + font-family: 'Open Sans', sans-serif; + font-weight: 700; + font-size: 1rem; + justify-content: center; + background-color: var(--main-control-color); + color: #fff; + cursor: pointer; + letter-spacing: 1.28px; + line-height: normal; + outline: none; + padding: 8px 18px; + text-align: center; + text-decoration: none; + border: 2px solid transparent; + border-radius: 32px; + text-transform: uppercase; + box-sizing: border-box; +} + +button:hover { + background-color: var(--main-control-color-hover); +} + +#refresh { + display: none; +} + +#start { + display: none; +} \ No newline at end of file From 3d0fb5e26af7cfc8b5e391b8388a57145971e79b Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 11:15:00 +0200 Subject: [PATCH 06/37] Working version for both --- .../Camera/extras/WebSerialCamera/app.js | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 01ce02425..3a00ddf22 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -25,9 +25,8 @@ let config = { const imageWidth = 320; // Adjust this value based on your bitmap width const imageHeight = 240; // Adjust this value based on your bitmap height -const bytesPerPixel = 2; // Adjust this value based on your bitmap format -// const mode = 'RGB565'; // Adjust this value based on your bitmap format -const mode = 'GRAYSCALE'; // Adjust this value based on your bitmap format +const mode = 'RGB565'; // Adjust this value based on your bitmap format +// const mode = 'GRAYSCALE'; // Adjust this value based on your bitmap format const totalBytes = imageWidth * imageHeight * config[mode].bytesPerPixel; // Set the buffer size to the total bytes. This allows to read the entire bitmap in one go. @@ -167,22 +166,21 @@ function renderBitmap(bytes, width, height) { canvas.height = height; const bytesPerPixel = config[mode].bytesPerPixel; const BYTES_PER_ROW = width * bytesPerPixel; - const BYTES_PER_COL = height * bytesPerPixel; const imageData = ctx.createImageData(canvas.width, canvas.height); - const data = imageData.data; + const dataContainer = imageData.data; - for (let row = 0; row < BYTES_PER_ROW; row++) { - for (let col = 0; col < BYTES_PER_COL; col++) { - const dataIndex = (row * BYTES_PER_ROW) + (col * bytesPerPixel); - const pixelValue = getPixelValue(bytes, dataIndex, bytesPerPixel); + for (let row = 0; row < height; row++) { + for (let col = 0; col < width; col++) { + const sourceDataIndex = (row * BYTES_PER_ROW) + (col * bytesPerPixel); + const pixelValue = getPixelValue(bytes, sourceDataIndex, bytesPerPixel); const [r, g, b] = config[mode].convert(pixelValue); - const idx = ((row * width) + col) * 4; - data[idx] = r; // Red channel - data[idx + 1] = g; // Green channel - data[idx + 2] = b; // Blue channel - data[idx + 3] = 255; // Alpha channel (opacity) + const pixelIndex = ((row * width) + col) * 4; + dataContainer[pixelIndex] = r; // Red channel + dataContainer[pixelIndex + 1] = g; // Green channel + dataContainer[pixelIndex + 2] = b; // Blue channel + dataContainer[pixelIndex + 3] = 255; // Alpha channel (opacity) } } ctx.clearRect(0, 0, canvas.width, canvas.height); From 599e764437ebd3ccd3c376abc9f3accd377fe4b6 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 14:09:29 +0200 Subject: [PATCH 07/37] Add image processor --- .../Camera/extras/WebSerialCamera/app.js | 50 ++--------- .../extras/WebSerialCamera/conversion.js | 19 ---- .../WebSerialCamera/imageDataProcessor.js | 86 +++++++++++++++++++ .../Camera/extras/WebSerialCamera/index.html | 2 +- 4 files changed, 94 insertions(+), 63 deletions(-) delete mode 100644 libraries/Camera/extras/WebSerialCamera/conversion.js create mode 100644 libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 3a00ddf22..c0dd54240 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -8,17 +8,14 @@ const ctx = canvas.getContext('2d'); const UserActionAbortError = 8; const ArduinoUSBVendorId = 0x2341; -let config = { - "RGB565": { - "convert": convertRGB565ToRGB888, +config = { + "RGB565": { "bytesPerPixel": 2 }, - "GRAYSCALE": { - "convert": convertGrayScaleToRGB888, + "GRAYSCALE": { "bytesPerPixel": 1 }, - "RGB888": { - "convert": convertToRGB888, + "RGB888": { "bytesPerPixel": 3 } }; @@ -37,6 +34,7 @@ const dataBits = 8; // Adjust this value based on your device's data bits const stopBits = 2; // Adjust this value based on your device's stop bits let currentPort, currentReader; +const imageDataProcessor = new ImageDataProcessor(ctx, mode, imageWidth, imageHeight); async function requestSerialPort(){ try { @@ -83,7 +81,6 @@ async function connectSerial(port, baudRate = 115200, dataBits = 8, stopBits = 2 } } - async function readBytes(port, numBytes, timeout = null){ if(port.readable.locked){ console.log('🔒 Stream is already locked. Ignoring request...'); @@ -145,44 +142,10 @@ async function readBytes(port, numBytes, timeout = null){ return bytesRead; } -// Get the pixel value using big endian -// Big-endian: the most significant byte comes first -function getPixelValue(data, index, bytesPerPixel){ - if(bytesPerPixel == 1){ - return data[index]; - } else if(bytesPerPixel == 2){ - return (data[index] << 8) | data[index + 1]; - } else if(bytesPerPixel == 3){ - return (data[index] << 16) | (data[index + 1] << 8) | data[index + 2]; - } else if(bytesPerPixel == 4){ - return (data[index] << 24) | (data[index + 1] << 16) | (data[index + 2] << 8) | data[index + 3]; - } - - return 0; -} - function renderBitmap(bytes, width, height) { canvas.width = width; canvas.height = height; - const bytesPerPixel = config[mode].bytesPerPixel; - const BYTES_PER_ROW = width * bytesPerPixel; - - const imageData = ctx.createImageData(canvas.width, canvas.height); - const dataContainer = imageData.data; - - for (let row = 0; row < height; row++) { - for (let col = 0; col < width; col++) { - const sourceDataIndex = (row * BYTES_PER_ROW) + (col * bytesPerPixel); - const pixelValue = getPixelValue(bytes, sourceDataIndex, bytesPerPixel); - const [r, g, b] = config[mode].convert(pixelValue); - - const pixelIndex = ((row * width) + col) * 4; - dataContainer[pixelIndex] = r; // Red channel - dataContainer[pixelIndex + 1] = g; // Green channel - dataContainer[pixelIndex + 2] = b; // Blue channel - dataContainer[pixelIndex + 3] = 255; // Alpha channel (opacity) - } - } + const imageData = imageDataProcessor.getImageDataBytes(bytes, width, height); ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.putImageData(imageData, 0, 0); } @@ -198,6 +161,7 @@ async function requestFrame(port){ await writer.write(new Uint8Array([1])); await writer.close(); } + async function renderStream(){ while(true && currentPort){ await renderFrame(currentPort); diff --git a/libraries/Camera/extras/WebSerialCamera/conversion.js b/libraries/Camera/extras/WebSerialCamera/conversion.js deleted file mode 100644 index ad467c077..000000000 --- a/libraries/Camera/extras/WebSerialCamera/conversion.js +++ /dev/null @@ -1,19 +0,0 @@ -function convertRGB565ToRGB888(pixelValue) { - // RGB565 - let r = (pixelValue >> (6 + 5)) & 0x1F; - let g = (pixelValue >> 5) & 0x3F; - let b = pixelValue & 0x1F; - // RGB888 - amplify - r <<= 3; - g <<= 2; - b <<= 3; - return [r, g, b]; -} - -function convertGrayScaleToRGB888(pixelValue) { - return [pixelValue, pixelValue, pixelValue]; -} - -function convertToRGB888(pixelValue){ - return [pixelValue[0], pixelValue[1], pixelValue[2]]; -} \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js new file mode 100644 index 000000000..3cf926b3b --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js @@ -0,0 +1,86 @@ +class ImageDataProcessor { + + constructor(context, mode) { + this.canvas = context.canvas; + this.context = context; + this.mode = mode; + this.config = { + "RGB565": { + "convert": this.convertRGB565ToRGB888, + "bytesPerPixel": 2 + }, + "GRAYSCALE": { + "convert": this.convertGrayScaleToRGB888, + "bytesPerPixel": 1 + }, + "RGB888": { + "convert": this.convertToRGB888, + "bytesPerPixel": 3 + } + }; + this.setMode(mode); + } + + setMode(mode) { + this.mode = mode; + this.bytesPerPixel = this.config[mode].bytesPerPixel; + } + + convertRGB565ToRGB888(pixelValue) { + // RGB565 + let r = (pixelValue >> (6 + 5)) & 0x1F; + let g = (pixelValue >> 5) & 0x3F; + let b = pixelValue & 0x1F; + // RGB888 - amplify + r <<= 3; + g <<= 2; + b <<= 3; + return [r, g, b]; + } + + convertGrayScaleToRGB888(pixelValue) { + return [pixelValue, pixelValue, pixelValue]; + } + + convertToRGB888(pixelValue){ + return [pixelValue[0], pixelValue[1], pixelValue[2]]; + } + + // Get the pixel value using big endian + // Big-endian: the most significant byte comes first + getPixelValue(sourceData, index) { + if (this.bytesPerPixel == 1) { + return sourceData[index]; + } else if (this.bytesPerPixel == 2) { + return (sourceData[index] << 8) | sourceData[index + 1]; + } else if (this.bytesPerPixel == 3) { + return (sourceData[index] << 16) | (sourceData[index + 1] << 8) | sourceData[index + 2]; + } else if (this.bytesPerPixel == 4) { + return (sourceData[index] << 24) | (sourceData[index + 1] << 16) | (sourceData[index + 2] << 8) | sourceData[index + 3]; + } + + return 0; + } + + getImageDataBytes(bytes, width, height) { + const BYTES_PER_ROW = width * this.bytesPerPixel; + + const imageData = this.context.createImageData(width, height); + const dataContainer = imageData.data; + + for (let row = 0; row < height; row++) { + for (let col = 0; col < width; col++) { + const sourceDataIndex = (row * BYTES_PER_ROW) + (col * this.bytesPerPixel); + const pixelValue = this.getPixelValue(bytes, sourceDataIndex, this.bytesPerPixel); + const [r, g, b] = this.config[mode].convert(pixelValue); + + const pixelIndex = ((row * width) + col) * 4; + dataContainer[pixelIndex] = r; // Red channel + dataContainer[pixelIndex + 1] = g; // Green channel + dataContainer[pixelIndex + 2] = b; // Blue channel + dataContainer[pixelIndex + 3] = 255; // Alpha channel (opacity) + } + } + return imageData; + } + } \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index 408aeac66..a6df69713 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -16,7 +16,7 @@ - + From 66f1a0206f9c79ec3fdbd1b4748f9e716992362b Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 14:09:36 +0200 Subject: [PATCH 08/37] Add save function --- libraries/Camera/extras/WebSerialCamera/app.js | 9 +++++++++ libraries/Camera/extras/WebSerialCamera/index.html | 1 + 2 files changed, 10 insertions(+) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index c0dd54240..2704dd9b9 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -2,6 +2,7 @@ const connectButton = document.getElementById('connect'); const refreshButton = document.getElementById('refresh'); const startButton = document.getElementById('start'); const disconnectButton = document.getElementById('disconnect'); +const saveImageButton = document.getElementById('save-image'); const canvas = document.getElementById('bitmapCanvas'); const ctx = canvas.getContext('2d'); @@ -211,6 +212,14 @@ refreshButton.addEventListener('click', () => { renderFrame(currentPort); }); +saveImageButton.addEventListener('click', () => { + const link = document.createElement('a'); + link.download = 'image.png'; + link.href = canvas.toDataURL(); + link.click(); + link.remove(); +}); + navigator.serial.addEventListener("connect", (e) => { // Connect to `e.target` or add it to a list of available ports. console.log('🔌 Serial port became available. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index a6df69713..a3de3033e 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -12,6 +12,7 @@
+
From 5ced73c22c868acdf23de88a6bd05ccd5057c183 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 15:17:15 +0200 Subject: [PATCH 09/37] Implement connection handler --- .../Camera/extras/WebSerialCamera/app.js | 173 ++---------------- .../Camera/extras/WebSerialCamera/index.html | 1 + .../serialConnectionHandler.js | 164 +++++++++++++++++ 3 files changed, 183 insertions(+), 155 deletions(-) create mode 100644 libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 2704dd9b9..66a76b91a 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -6,8 +6,11 @@ const saveImageButton = document.getElementById('save-image'); const canvas = document.getElementById('bitmapCanvas'); const ctx = canvas.getContext('2d'); -const UserActionAbortError = 8; -const ArduinoUSBVendorId = 0x2341; +// TODO check for signals +// TODO implement transformer +// TODO get image format from device +// SEE: https://developer.chrome.com/articles/serial/#transforming-streams +// SEE: https://developer.chrome.com/articles/serial/#signals config = { "RGB565": { @@ -34,114 +37,8 @@ const baudRate = 115200; // Adjust this value based on your device's baud rate const dataBits = 8; // Adjust this value based on your device's data bits const stopBits = 2; // Adjust this value based on your device's stop bits -let currentPort, currentReader; const imageDataProcessor = new ImageDataProcessor(ctx, mode, imageWidth, imageHeight); - -async function requestSerialPort(){ - try { - // Request a serial port - const port = await navigator.serial.requestPort({ filters: [{ usbVendorId: ArduinoUSBVendorId }] }); - currentPort = port; - return port; - } catch (error) { - if(error.code != UserActionAbortError){ - console.log(error); - } - return null; - } -} - -async function autoConnect(){ - if(currentPort){ - console.log('🔌 Already connected to a serial port.'); - return false; - } - - // Get all serial ports the user has previously granted the website access to. - const ports = await navigator.serial.getPorts(); - - for (const port of ports) { - console.log('👀 Serial port found with VID: 0x' + port.getInfo().usbVendorId.toString(16)); - if(port.getInfo().usbVendorId === ArduinoUSBVendorId){ - currentPort = port; - return await connectSerial(currentPort); - } - } - return false; -} - -async function connectSerial(port, baudRate = 115200, dataBits = 8, stopBits = 2, bufferSize = 4096) { - try { - // If the port is already open, close it - if (port.readable) await port.close(); - await port.open({ baudRate: baudRate, parity: "even", dataBits: dataBits, stopBits: stopBits, bufferSize: bufferSize }); - console.log('✅ Connected to serial port.'); - return true; - } catch (error) { - return false; - } -} - -async function readBytes(port, numBytes, timeout = null){ - if(port.readable.locked){ - console.log('🔒 Stream is already locked. Ignoring request...'); - return null; - } - - const bytesRead = new Uint8Array(numBytes); - let bytesReadIdx = 0; - let keepReading = true; - - // As long as the errors are non-fatal, a new ReadableStream is created automatically and hence port.readable is non-null. - // If a fatal error occurs, such as the serial device being removed, then port.readable becomes null. - - while (port.readable && keepReading) { - const reader = port.readable.getReader(); - currentReader = reader; - let timeoutID = null; - // let count = 0; - - try { - while (bytesReadIdx < numBytes) { - if(timeout){ - timeoutID = setTimeout(() => { - console.log('⌛️ Timeout occurred while reading.'); - if(port.readable) reader?.cancel(); - }, timeout); - } - - const { value, done } = await reader.read(); - if(timeoutID) clearTimeout(timeoutID); - - if(value){ - for (const byte of value) { - bytesRead[bytesReadIdx++] = byte; - if (bytesReadIdx >= numBytes) break; - } - // count += value.byteLength; - // console.log(`Read ${value.byteLength} (Total: ${count}) out of ${numBytes} bytes.}`); - } - - if (done) { - // |reader| has been canceled. - console.log('🚫 Reader has been canceled'); - break; - } - } - - } catch (error) { - // Handle |error|... - console.log('💣 Error occurred while reading: '); - console.log(error); - } finally { - keepReading = false; - // console.log('🔓 Releasing reader lock...'); - reader?.releaseLock(); - currentReader = null; - } - } - return bytesRead; -} +const connectionHandler = new SerialConnectionHandler(baudRate, dataBits, stopBits, "even", "hardware", bufferSize); function renderBitmap(bytes, width, height) { canvas.width = width; @@ -151,65 +48,31 @@ function renderBitmap(bytes, width, height) { ctx.putImageData(imageData, 0, 0); } -async function requestFrame(port){ - if(!port?.writable) { - console.log('🚫 Port is not writable. Ignoring request...'); - return; - } - // console.log('Writing 1 to the serial port...'); - // Write a 1 to the serial port - const writer = port.writable.getWriter(); - await writer.write(new Uint8Array([1])); - await writer.close(); -} - async function renderStream(){ - while(true && currentPort){ - await renderFrame(currentPort); + while(connectionHandler.isConnected()){ + await renderFrame(); } } -async function renderFrame(port){ - if(!port) return; - const bytes = await getFrame(port); - if(!bytes) return false; // Nothing to render +async function renderFrame(){ + if(!connectionHandler.isConnected()) return; + const bytes = await connectionHandler.getFrame(totalBytes); + if(!bytes || bytes.length == 0) return false; // Nothing to render // console.log(`Reading done ✅. Rendering image...`); - // Render the bytes as a grayscale bitmap renderBitmap(bytes, imageWidth, imageHeight); return true; } -async function getFrame(port) { - if(!port) return; - - await requestFrame(port); - // console.log(`Trying to read ${totalBytes} bytes...`); - // Read the given amount of bytes - return await readBytes(port, totalBytes, 2000); -} - -async function disconnectSerial(port) { - if(!port) return; - try { - currentPort = null; - await currentReader?.cancel(); - await port.close(); - console.log('🔌 Disconnected from serial port.'); - } catch (error) { - console.error('💣 Error occurred while disconnecting: ' + error.message); - }; -} - startButton.addEventListener('click', renderStream); connectButton.addEventListener('click', async () => { - currentPort = await requestSerialPort(); - if(await connectSerial(currentPort, baudRate, dataBits, stopBits, bufferSize, flowControl)){ + await connectionHandler.requestSerialPort(); + if(await connectionHandler.connectSerial()){ renderStream(); } }); -disconnectButton.addEventListener('click', () => disconnectSerial(currentPort)); +disconnectButton.addEventListener('click', () => connectionHandler.disconnectSerial()); refreshButton.addEventListener('click', () => { - renderFrame(currentPort); + renderFrame(); }); saveImageButton.addEventListener('click', () => { @@ -223,7 +86,7 @@ saveImageButton.addEventListener('click', () => { navigator.serial.addEventListener("connect", (e) => { // Connect to `e.target` or add it to a list of available ports. console.log('🔌 Serial port became available. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); - autoConnect().then((connected) => { + connectionHandler.autoConnect().then((connected) => { if(connected){ renderStream(); }; @@ -240,7 +103,7 @@ navigator.serial.addEventListener("disconnect", (e) => { window.addEventListener('load', async () => { console.log('🚀 Page loaded. Trying to connect to serial port...'); setTimeout(() => { - autoConnect().then((connected) => { + connectionHandler.autoConnect().then((connected) => { if (connected) { renderStream(); }; diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index a3de3033e..0d4dad07b 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -18,6 +18,7 @@ + diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js new file mode 100644 index 000000000..acf2d267b --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -0,0 +1,164 @@ +const ArduinoUSBVendorId = 0x2341; +const UserActionAbortError = 8; + +class SerialConnectionHandler { + constructor(baudRate = 115200, dataBits = 8, stopBits = 1, parity = "none", flowControl = "none", bufferSize = 4096, timeout = 2000) { + this.baudRate = baudRate; + this.dataBits = dataBits; + this.stopBits = stopBits; + this.flowControl = flowControl; + this.bufferSize = bufferSize; + this.parity = parity; + this.timeout = timeout; + this.currentPort = null; + this.currentReader = null; + } + + async requestSerialPort() { + try { + const port = await navigator.serial.requestPort({ filters: [{ usbVendorId: ArduinoUSBVendorId }] }); + this.currentPort = port; + return port; + } catch (error) { + if (error.code != UserActionAbortError) { + console.log(error); + } + return null; + } + } + + isConnected(){ + return this.currentPort?.readable != null; + } + + async connectSerial() { + try { + // If the port is already open, close it + if (this.isConnected()) await this.currentPort.close(); + await this.currentPort.open({ + baudRate: this.baudRate, + parity: this.parity, + dataBits: this.dataBits, + stopBits: this.stopBits, + bufferSize: this.bufferSize, + flowControl: this.flowControl + }); + console.log('✅ Connected to serial port.'); + return true; + } catch (error) { + return false; + } + } + + async disconnectSerial() { + if (!this.currentPort) return; + try { + const port = this.currentPort; + this.currentPort = null; + await this.currentReader?.cancel(); + await port.close(); + console.log('🔌 Disconnected from serial port.'); + } catch (error) { + console.error('💣 Error occurred while disconnecting: ' + error.message); + }; + } + + async autoConnect() { + if (this.currentPort) { + console.log('🔌 Already connected to a serial port.'); + return false; + } + + // Get all serial ports the user has previously granted the website access to. + const ports = await navigator.serial.getPorts(); + + for (const port of ports) { + console.log('👀 Serial port found with VID: 0x' + port.getInfo().usbVendorId.toString(16)); + if (port.getInfo().usbVendorId === ArduinoUSBVendorId) { + this.currentPort = port; + return await this.connectSerial(this.currentPort); + } + } + return false; + } + + async readBytes(numBytes, timeout = null) { + if (this.currentPort.readable.locked) { + console.log('🔒 Stream is already locked. Ignoring request...'); + return null; + } + + const bytesRead = new Uint8Array(numBytes); + let bytesReadIdx = 0; + let keepReading = true; + + // As long as the errors are non-fatal, a new ReadableStream is created automatically and hence port.readable is non-null. + // If a fatal error occurs, such as the serial device being removed, then port.readable becomes null. + + while (this.currentPort?.readable && keepReading) { + const reader = this.currentPort.readable.getReader(); + this.currentReader = reader; + let timeoutID = null; + // let count = 0; + + try { + while (bytesReadIdx < numBytes) { + if (timeout) { + timeoutID = setTimeout(() => { + console.log('⌛️ Timeout occurred while reading.'); + if (this.currentPort.readable) reader?.cancel(); + }, timeout); + } + + const { value, done } = await reader.read(); + if (timeoutID) clearTimeout(timeoutID); + + if (value) { + for (const byte of value) { + bytesRead[bytesReadIdx++] = byte; + if (bytesReadIdx >= numBytes) break; + } + // count += value.byteLength; + // console.log(`Read ${value.byteLength} (Total: ${count}) out of ${numBytes} bytes.}`); + } + + if (done) { + console.log('🚫 Reader has been canceled'); + break; + } + } + + } catch (error) { + console.log('💣 Error occurred while reading: '); + console.log(error); + } finally { + keepReading = false; + // console.log('🔓 Releasing reader lock...'); + reader?.releaseLock(); + this.currentReader = null; + } + } + return bytesRead; + } + + async requestFrame(){ + if(!this.currentPort?.writable) { + console.log('🚫 Port is not writable. Ignoring request...'); + return; + } + // console.log('Writing 1 to the serial port...'); + // Write a 1 to the serial port + const writer = this.currentPort.writable.getWriter(); + await writer.write(new Uint8Array([1])); + await writer.close(); + } + + async getFrame(totalBytes) { + if (!this.currentPort) return; + + await this.requestFrame(this.currentPort); + // console.log(`Trying to read ${totalBytes} bytes...`); + // Read the given amount of bytes + return await this.readBytes(totalBytes, this.timeout); + } +} \ No newline at end of file From e2789f425747ae6af26e030123d076409b6f7a6b Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 16:08:08 +0200 Subject: [PATCH 10/37] Add callbacks for connect / disconnect --- .../Camera/extras/WebSerialCamera/app.js | 41 +++++++------------ .../Camera/extras/WebSerialCamera/index.html | 1 - .../serialConnectionHandler.js | 34 +++++++++++---- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 66a76b91a..eb1f7cbc2 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -1,7 +1,6 @@ const connectButton = document.getElementById('connect'); const refreshButton = document.getElementById('refresh'); const startButton = document.getElementById('start'); -const disconnectButton = document.getElementById('disconnect'); const saveImageButton = document.getElementById('save-image'); const canvas = document.getElementById('bitmapCanvas'); const ctx = canvas.getContext('2d'); @@ -40,6 +39,15 @@ const stopBits = 2; // Adjust this value based on your device's stop bits const imageDataProcessor = new ImageDataProcessor(ctx, mode, imageWidth, imageHeight); const connectionHandler = new SerialConnectionHandler(baudRate, dataBits, stopBits, "even", "hardware", bufferSize); +connectionHandler.onConnect = () => { + connectButton.textContent = 'Disconnect'; + renderStream(); +}; + +connectionHandler.onDisconnect = () => { + connectButton.textContent = 'Connect'; +}; + function renderBitmap(bytes, width, height) { canvas.width = width; canvas.height = height; @@ -65,12 +73,13 @@ async function renderFrame(){ startButton.addEventListener('click', renderStream); connectButton.addEventListener('click', async () => { - await connectionHandler.requestSerialPort(); - if(await connectionHandler.connectSerial()){ - renderStream(); + if(connectionHandler.isConnected()){ + connectionHandler.disconnectSerial(); + } else { + await connectionHandler.requestSerialPort(); + await connectionHandler.connectSerial(); } }); -disconnectButton.addEventListener('click', () => connectionHandler.disconnectSerial()); refreshButton.addEventListener('click', () => { renderFrame(); }); @@ -83,31 +92,11 @@ saveImageButton.addEventListener('click', () => { link.remove(); }); -navigator.serial.addEventListener("connect", (e) => { - // Connect to `e.target` or add it to a list of available ports. - console.log('🔌 Serial port became available. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); - connectionHandler.autoConnect().then((connected) => { - if(connected){ - renderStream(); - }; - }); -}); - -navigator.serial.addEventListener("disconnect", (e) => { - // Remove `e.target` from the list of available ports. - console.log('❌ Serial port lost. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); - currentPort = null; -}); - // On page load event, try to connect to the serial port window.addEventListener('load', async () => { console.log('🚀 Page loaded. Trying to connect to serial port...'); setTimeout(() => { - connectionHandler.autoConnect().then((connected) => { - if (connected) { - renderStream(); - }; - }); + connectionHandler.autoConnect(); }, 1000); }); diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index 0d4dad07b..5d71c4bc2 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -11,7 +11,6 @@
- diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index acf2d267b..cea6667b8 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -12,6 +12,7 @@ class SerialConnectionHandler { this.timeout = timeout; this.currentPort = null; this.currentReader = null; + this.registerEvents(); } async requestSerialPort() { @@ -27,7 +28,7 @@ class SerialConnectionHandler { } } - isConnected(){ + isConnected() { return this.currentPort?.readable != null; } @@ -44,6 +45,7 @@ class SerialConnectionHandler { flowControl: this.flowControl }); console.log('✅ Connected to serial port.'); + if(this.onConnect) this.onConnect(); return true; } catch (error) { return false; @@ -58,6 +60,7 @@ class SerialConnectionHandler { await this.currentReader?.cancel(); await port.close(); console.log('🔌 Disconnected from serial port.'); + if(this.onDisconnect) this.onDisconnect(); } catch (error) { console.error('💣 Error occurred while disconnecting: ' + error.message); }; @@ -106,7 +109,7 @@ class SerialConnectionHandler { if (timeout) { timeoutID = setTimeout(() => { console.log('⌛️ Timeout occurred while reading.'); - if (this.currentPort.readable) reader?.cancel(); + if (this.currentPort?.readable) reader?.cancel(); }, timeout); } @@ -129,8 +132,7 @@ class SerialConnectionHandler { } } catch (error) { - console.log('💣 Error occurred while reading: '); - console.log(error); + console.log('💣 Error occurred while reading: ' + error.message); } finally { keepReading = false; // console.log('🔓 Releasing reader lock...'); @@ -141,17 +143,17 @@ class SerialConnectionHandler { return bytesRead; } - async requestFrame(){ - if(!this.currentPort?.writable) { - console.log('🚫 Port is not writable. Ignoring request...'); - return; + async requestFrame() { + if (!this.currentPort?.writable) { + console.log('🚫 Port is not writable. Ignoring request...'); + return; } // console.log('Writing 1 to the serial port...'); // Write a 1 to the serial port const writer = this.currentPort.writable.getWriter(); await writer.write(new Uint8Array([1])); await writer.close(); - } + } async getFrame(totalBytes) { if (!this.currentPort) return; @@ -161,4 +163,18 @@ class SerialConnectionHandler { // Read the given amount of bytes return await this.readBytes(totalBytes, this.timeout); } + + registerEvents() { + navigator.serial.addEventListener("connect", (e) => { + // Connect to `e.target` or add it to a list of available ports. + console.log('🔌 Serial port became available. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); + this.autoConnect(); + }); + + navigator.serial.addEventListener("disconnect", (e) => { + console.log('❌ Serial port lost. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); + this.currentPort = null; + if(this.onDisconnect) this.onDisconnect(); + }); + } } \ No newline at end of file From a85053c49f856c705ae0c6df06053cc1fc5038d8 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 24 May 2023 17:38:16 +0200 Subject: [PATCH 11/37] Add documentation to serial connection handler --- .../serialConnectionHandler.js | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index cea6667b8..1c0be9f25 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -1,6 +1,9 @@ const ArduinoUSBVendorId = 0x2341; const UserActionAbortError = 8; +/** + * Handles the connection between the browser and the Arduino board via Web Serial. + */ class SerialConnectionHandler { constructor(baudRate = 115200, dataBits = 8, stopBits = 1, parity = "none", flowControl = "none", bufferSize = 4096, timeout = 2000) { this.baudRate = baudRate; @@ -15,6 +18,10 @@ class SerialConnectionHandler { this.registerEvents(); } + /** + * Prompts the user to select a serial port. + * @returns {Promise} The serial port that the user has selected. + */ async requestSerialPort() { try { const port = await navigator.serial.requestPort({ filters: [{ usbVendorId: ArduinoUSBVendorId }] }); @@ -28,10 +35,20 @@ class SerialConnectionHandler { } } + /** + * Checks if the browser is connected to a serial port. + * @returns {boolean} True if the browser is connected, false otherwise. + */ isConnected() { return this.currentPort?.readable != null; } + /** + * Opens a connection to the given serial port by using the settings specified in the constructor. + * If the port is already open, it will be closed first. + * This method will call the `onConnect` callback before it returns. + * @returns {boolean} True if the connection was successfully opened, false otherwise. + */ async connectSerial() { try { // If the port is already open, close it @@ -52,6 +69,12 @@ class SerialConnectionHandler { } } + /** + * Disconnects from the current serial port. + * If a reading operation is in progress, it will be canceled. + * This function will call the `onDisconnect` callback before it returns. + * @returns {Promise} A promise that resolves when the port has been closed. + */ async disconnectSerial() { if (!this.currentPort) return; try { @@ -66,6 +89,11 @@ class SerialConnectionHandler { }; } + /** + * Tries to establish a connection to the first available serial port that has the Arduino USB vendor ID. + * This only works if the user has previously granted the website access to that serial port. + * @returns {Promise} True if the connection was successfully opened, false otherwise. + */ async autoConnect() { if (this.currentPort) { console.log('🔌 Already connected to a serial port.'); @@ -85,6 +113,12 @@ class SerialConnectionHandler { return false; } + /** + * Reads the specified number of bytes from the serial port. + * @param {number} numBytes The number of bytes to read. + * @param {number} timeout The timeout in milliseconds. + * If the timeout is reached, the reader will be canceled and the read lock will be released. + */ async readBytes(numBytes, timeout = null) { if (this.currentPort.readable.locked) { console.log('🔒 Stream is already locked. Ignoring request...'); @@ -143,6 +177,10 @@ class SerialConnectionHandler { return bytesRead; } + /** + * Reqests an image frame from the Arduino board by writing a 1 to the serial port. + * @returns {Promise} A promise that resolves when the frame has been requested and the write stream has been closed. + */ async requestFrame() { if (!this.currentPort?.writable) { console.log('🚫 Port is not writable. Ignoring request...'); @@ -155,6 +193,11 @@ class SerialConnectionHandler { await writer.close(); } + /** + * Requests a frame from the Arduino board and reads the specified number of bytes from the serial port afterwards. + * Times out after the timeout in milliseconds specified in the constructor. + * @param {number} totalBytes The number of bytes to read. + */ async getFrame(totalBytes) { if (!this.currentPort) return; @@ -164,6 +207,13 @@ class SerialConnectionHandler { return await this.readBytes(totalBytes, this.timeout); } + /** + * Registers event listeners for the `connect` and `disconnect` events of the serial port. + * The `connect` event is fired when a serial port becomes available not when it is opened. + * When the `connect` event is fired, `autoConnect()` is called. + * The `disconnect` event is fired when a serial port is lost. + * When the `disconnect` event is fired, the `onDisconnect` callback is called. + **/ registerEvents() { navigator.serial.addEventListener("connect", (e) => { // Connect to `e.target` or add it to a list of available ports. From fe1cef19228e0a5ed424828a71e45a3b89e01e1f Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 26 May 2023 10:02:49 +0200 Subject: [PATCH 12/37] Better error reporting --- .../Camera/extras/WebSerialCamera/serialConnectionHandler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index 1c0be9f25..653ddc2a5 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -166,7 +166,7 @@ class SerialConnectionHandler { } } catch (error) { - console.log('💣 Error occurred while reading: ' + error.message); + console.error('💣 Error occurred while reading: ' + error.message); } finally { keepReading = false; // console.log('🔓 Releasing reader lock...'); From a7a35a0cb94aaf0086409392f56a3a438d7d8949 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 30 Jun 2023 12:30:10 +0200 Subject: [PATCH 13/37] Change order of config bytes --- .../CameraCaptureRawBytes/CameraCaptureRawBytes.ino | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino index c4c88ec63..3cd988a70 100644 --- a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino +++ b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino @@ -6,10 +6,11 @@ Camera cam(galaxyCore); #define IMAGE_MODE CAMERA_RGB565 #elif defined(ARDUINO_PORTENTA_H7_M7) - //#include "hm0360.h" - //HM0360 himax; - #include "himax.h"; - HM01B0 himax; + // uncomment the correct camera in use + #include "hm0360.h" + HM0360 himax; + // #include "himax.h"; + // HM01B0 himax; Camera cam(himax); #define IMAGE_MODE CAMERA_GRAYSCALE #elif defined(ARDUINO_GIGA) @@ -87,8 +88,8 @@ void sendFrame(){ } void sendCameraConfig(){ - Serial.write(RESOLUTION); Serial.write(IMAGE_MODE); + Serial.write(RESOLUTION); Serial.flush(); delay(1); } From 23136363ea1d3e6be5a6aab1821918aaad896581 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 30 Jun 2023 12:30:38 +0200 Subject: [PATCH 14/37] Add auto config for camera --- .../Camera/extras/WebSerialCamera/app.js | 45 ++++------- .../extras/WebSerialCamera/cameraConfig.js | 38 +++++++++ .../WebSerialCamera/imageDataProcessor.js | 79 ++++++++++++------- .../Camera/extras/WebSerialCamera/index.html | 1 + .../serialConnectionHandler.js | 32 ++++++-- 5 files changed, 132 insertions(+), 63 deletions(-) create mode 100644 libraries/Camera/extras/WebSerialCamera/cameraConfig.js diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index eb1f7cbc2..699f09ef5 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -11,63 +11,52 @@ const ctx = canvas.getContext('2d'); // SEE: https://developer.chrome.com/articles/serial/#transforming-streams // SEE: https://developer.chrome.com/articles/serial/#signals -config = { - "RGB565": { - "bytesPerPixel": 2 - }, - "GRAYSCALE": { - "bytesPerPixel": 1 - }, - "RGB888": { - "bytesPerPixel": 3 - } -}; - -const imageWidth = 320; // Adjust this value based on your bitmap width -const imageHeight = 240; // Adjust this value based on your bitmap height -const mode = 'RGB565'; // Adjust this value based on your bitmap format -// const mode = 'GRAYSCALE'; // Adjust this value based on your bitmap format -const totalBytes = imageWidth * imageHeight * config[mode].bytesPerPixel; // Set the buffer size to the total bytes. This allows to read the entire bitmap in one go. -const bufferSize = Math.min(totalBytes, 16 * 1024 * 1024); // Max buffer size is 16MB +const bufferSize = 1024 * 1024;//Math.min(totalBytes, 16 * 1024 * 1024); // Max buffer size is 16MB const flowControl = 'hardware'; const baudRate = 115200; // Adjust this value based on your device's baud rate const dataBits = 8; // Adjust this value based on your device's data bits const stopBits = 2; // Adjust this value based on your device's stop bits -const imageDataProcessor = new ImageDataProcessor(ctx, mode, imageWidth, imageHeight); +const imageDataProcessor = new ImageDataProcessor(ctx); const connectionHandler = new SerialConnectionHandler(baudRate, dataBits, stopBits, "even", "hardware", bufferSize); -connectionHandler.onConnect = () => { +connectionHandler.onConnect = async () => { connectButton.textContent = 'Disconnect'; + cameraConfig = await connectionHandler.getConfig(); + const imageMode = CAMERA_MODES[cameraConfig[0]]; + const imageResolution = CAMERA_RESOLUTIONS[cameraConfig[1]]; + imageDataProcessor.setMode(imageMode); + imageDataProcessor.setResolution(imageResolution.width, imageResolution.height); renderStream(); }; connectionHandler.onDisconnect = () => { connectButton.textContent = 'Connect'; + imageDataProcessor.reset(); }; -function renderBitmap(bytes, width, height) { - canvas.width = width; - canvas.height = height; - const imageData = imageDataProcessor.getImageDataBytes(bytes, width, height); +function renderBitmap(imageData) { + canvas.width = imageDataProcessor.width; + canvas.height = imageDataProcessor.height; ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.putImageData(imageData, 0, 0); } async function renderStream(){ while(connectionHandler.isConnected()){ - await renderFrame(); + if(imageDataProcessor.isConfigured()) await renderFrame(); } } async function renderFrame(){ if(!connectionHandler.isConnected()) return; - const bytes = await connectionHandler.getFrame(totalBytes); + const bytes = await connectionHandler.getFrame(imageDataProcessor.getTotalBytes()); if(!bytes || bytes.length == 0) return false; // Nothing to render // console.log(`Reading done ✅. Rendering image...`); - renderBitmap(bytes, imageWidth, imageHeight); + const imageData = imageDataProcessor.getImageData(bytes); + renderBitmap(imageData); return true; } @@ -81,7 +70,7 @@ connectButton.addEventListener('click', async () => { } }); refreshButton.addEventListener('click', () => { - renderFrame(); + if(imageDataProcessor.isConfigured()) renderFrame(); }); saveImageButton.addEventListener('click', () => { diff --git a/libraries/Camera/extras/WebSerialCamera/cameraConfig.js b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js new file mode 100644 index 000000000..d3a31286b --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js @@ -0,0 +1,38 @@ +const CAMERA_MODES = { + 0: "GRAYSCALE", + 1: "BAYER", + 2: "RGB565" +}; + +const CAMERA_RESOLUTIONS = { + 0: { + "name": "QQVGA", + "width": 160, + "height": 120 + }, + 1: { + "name": "QVGA", + "width": 320, + "height": 240 + }, + 2: { + "name": "320x320", + "width": 320, + "height": 320 + }, + 3: { + "name": "VGA", + "width": 640, + "height": 480 + }, + 5: { + "name": "SVGA", + "width": 800, + "height": 600 + }, + 6: { + "name": "UXGA", + "width": 1600, + "height": 1200 + } +}; \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js index 3cf926b3b..ad1a8fdf0 100644 --- a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js +++ b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js @@ -1,29 +1,54 @@ class ImageDataProcessor { + pixelFormatInfo = { + "RGB565": { + "convert": this.convertRGB565ToRGB888, + "bytesPerPixel": 2 + }, + "GRAYSCALE": { + "convert": this.convertGrayScaleToRGB888, + "bytesPerPixel": 1 + }, + "RGB888": { + "convert": this.convertToRGB888, + "bytesPerPixel": 3 + }, + "BAYER": { + "convert": null, // TODO + "bytesPerPixel": 1 + } + }; - constructor(context, mode) { - this.canvas = context.canvas; - this.context = context; - this.mode = mode; - this.config = { - "RGB565": { - "convert": this.convertRGB565ToRGB888, - "bytesPerPixel": 2 - }, - "GRAYSCALE": { - "convert": this.convertGrayScaleToRGB888, - "bytesPerPixel": 1 - }, - "RGB888": { - "convert": this.convertToRGB888, - "bytesPerPixel": 3 - } - }; - this.setMode(mode); + constructor(context, mode = null, width = null, height = null) { + this.context = context; + this.canvas = context.canvas; + + if(mode) this.setMode(mode); + if(width && height) this.setResolution(width, height); } setMode(mode) { this.mode = mode; - this.bytesPerPixel = this.config[mode].bytesPerPixel; + this.bytesPerPixel = this.pixelFormatInfo[mode].bytesPerPixel; + } + + setResolution(width, height) { + this.width = width; + this.height = height; + } + + getTotalBytes() { + return this.width * this.height * this.bytesPerPixel; + } + + isConfigured() { + return this.mode && this.width && this.height; + } + + reset() { + this.mode = null; + this.bytesPerPixel = null; + this.width = null; + this.height = null; } convertRGB565ToRGB888(pixelValue) { @@ -62,19 +87,19 @@ class ImageDataProcessor { return 0; } - getImageDataBytes(bytes, width, height) { - const BYTES_PER_ROW = width * this.bytesPerPixel; + getImageData(bytes) { + const BYTES_PER_ROW = this.width * this.bytesPerPixel; - const imageData = this.context.createImageData(width, height); + const imageData = this.context.createImageData(this.width, this.height); const dataContainer = imageData.data; - for (let row = 0; row < height; row++) { - for (let col = 0; col < width; col++) { + for (let row = 0; row < this.height; row++) { + for (let col = 0; col < this.width; col++) { const sourceDataIndex = (row * BYTES_PER_ROW) + (col * this.bytesPerPixel); const pixelValue = this.getPixelValue(bytes, sourceDataIndex, this.bytesPerPixel); - const [r, g, b] = this.config[mode].convert(pixelValue); + const [r, g, b] = this.pixelFormatInfo[this.mode].convert(pixelValue); - const pixelIndex = ((row * width) + col) * 4; + const pixelIndex = ((row * this.width) + col) * 4; // 4 channels: R, G, B, A dataContainer[pixelIndex] = r; // Red channel dataContainer[pixelIndex + 1] = g; // Green channel dataContainer[pixelIndex + 2] = b; // Blue channel diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index 5d71c4bc2..91a5c1147 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -18,6 +18,7 @@
+ diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index 653ddc2a5..ae0a024be 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -177,20 +177,36 @@ class SerialConnectionHandler { return bytesRead; } + async sendData(byteArray) { + if (!this.currentPort?.writable) { + console.log('🚫 Port is not writable. Ignoring request...'); + return; + } + const writer = this.currentPort.writable.getWriter(); + await writer.write(new Uint8Array(byteArray)); + await writer.close(); + } + /** * Reqests an image frame from the Arduino board by writing a 1 to the serial port. * @returns {Promise} A promise that resolves when the frame has been requested and the write stream has been closed. */ async requestFrame() { - if (!this.currentPort?.writable) { - console.log('🚫 Port is not writable. Ignoring request...'); - return; - } // console.log('Writing 1 to the serial port...'); // Write a 1 to the serial port - const writer = this.currentPort.writable.getWriter(); - await writer.write(new Uint8Array([1])); - await writer.close(); + return this.sendData([1]); + } + + async requestConfig() { + return this.sendData([2]); + } + + async getConfig() { + if (!this.currentPort) return; + + await this.requestConfig(); + // console.log(`Trying to read 2 bytes...`); + return await this.readBytes(2, this.timeout); } /** @@ -201,7 +217,7 @@ class SerialConnectionHandler { async getFrame(totalBytes) { if (!this.currentPort) return; - await this.requestFrame(this.currentPort); + await this.requestFrame(); // console.log(`Trying to read ${totalBytes} bytes...`); // Read the given amount of bytes return await this.readBytes(totalBytes, this.timeout); From fb8a54f435dd681b1ce5d79a7dfe920ecef9e951 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Dec 2023 10:06:03 +0100 Subject: [PATCH 15/37] Better error handling --- libraries/Camera/extras/WebSerialCamera/app.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 699f09ef5..59866edaa 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -25,8 +25,16 @@ const connectionHandler = new SerialConnectionHandler(baudRate, dataBits, stopBi connectionHandler.onConnect = async () => { connectButton.textContent = 'Disconnect'; cameraConfig = await connectionHandler.getConfig(); + if(!cameraConfig){ + console.error('🚫 Could not read camera configuration. Aborting...'); + return; + } const imageMode = CAMERA_MODES[cameraConfig[0]]; const imageResolution = CAMERA_RESOLUTIONS[cameraConfig[1]]; + if(!imageMode || !imageResolution){ + console.error(`🚫 Invalid camera configuration: ${cameraConfig[0]}, ${cameraConfig[1]}. Aborting...`); + return; + } imageDataProcessor.setMode(imageMode); imageDataProcessor.setResolution(imageResolution.width, imageResolution.height); renderStream(); From 7149a1b6b2091947b513a5f4d99532dca264c764 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Dec 2023 10:06:23 +0100 Subject: [PATCH 16/37] Extract magic strings --- libraries/Camera/extras/WebSerialCamera/app.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 59866edaa..bb6922d2e 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -13,14 +13,15 @@ const ctx = canvas.getContext('2d'); // Set the buffer size to the total bytes. This allows to read the entire bitmap in one go. -const bufferSize = 1024 * 1024;//Math.min(totalBytes, 16 * 1024 * 1024); // Max buffer size is 16MB +const bufferSize = 2 * 1024 * 1024; // Max buffer size is 16MB const flowControl = 'hardware'; const baudRate = 115200; // Adjust this value based on your device's baud rate const dataBits = 8; // Adjust this value based on your device's data bits const stopBits = 2; // Adjust this value based on your device's stop bits +const parityBit = 'even'; // Adjust this value based on your device's parity bit const imageDataProcessor = new ImageDataProcessor(ctx); -const connectionHandler = new SerialConnectionHandler(baudRate, dataBits, stopBits, "even", "hardware", bufferSize); +const connectionHandler = new SerialConnectionHandler(baudRate, dataBits, stopBits, parityBit, flowControl, bufferSize); connectionHandler.onConnect = async () => { connectButton.textContent = 'Disconnect'; From e2e949b54de3689d3d0715d11c3048652aaae637 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Dec 2023 10:07:23 +0100 Subject: [PATCH 17/37] Add references --- libraries/Camera/extras/WebSerialCamera/app.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index bb6922d2e..89497ebb4 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -5,12 +5,9 @@ const saveImageButton = document.getElementById('save-image'); const canvas = document.getElementById('bitmapCanvas'); const ctx = canvas.getContext('2d'); -// TODO check for signals -// TODO implement transformer -// TODO get image format from device -// SEE: https://developer.chrome.com/articles/serial/#transforming-streams -// SEE: https://developer.chrome.com/articles/serial/#signals - +// Check the following links for more information on the Web Serial API: +// https://developer.chrome.com/articles/serial/ +// https://wicg.github.io/serial/ // Set the buffer size to the total bytes. This allows to read the entire bitmap in one go. const bufferSize = 2 * 1024 * 1024; // Max buffer size is 16MB From 815151d9a9ef30afcbaf58ae54a49944da3d00ad Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Dec 2023 11:01:52 +0100 Subject: [PATCH 18/37] Stream works, disconnect broken --- .../serialConnectionHandler.js | 116 ++++++++++-------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index ae0a024be..552fe35c6 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -1,6 +1,43 @@ const ArduinoUSBVendorId = 0x2341; const UserActionAbortError = 8; +class BytesWaitTransformer { + constructor(waitBytes) { + this.waitBytes = waitBytes; + this.buffer = new Uint8Array(0); + this.controller = undefined; + } + + async transform(chunk, controller) { + this.controller = controller; + + // Concatenate incoming chunk with existing buffer + this.buffer = new Uint8Array([...this.buffer, ...chunk]); + + while (this.buffer.length >= this.waitBytes) { + // Extract the required number of bytes + const bytesToProcess = this.buffer.slice(0, this.waitBytes); + + // Remove processed bytes from the buffer + this.buffer = this.buffer.slice(this.waitBytes); + + // Notify the controller that bytes have been processed + controller.enqueue(bytesToProcess); + } + } + + flush(controller) { + if (this.buffer.length > 0) { + // Handle remaining bytes (if any) when the stream is closed + const remainingBytes = this.buffer.slice(); + console.log("Remaining bytes:", remainingBytes); + + // Notify the controller that remaining bytes have been processed + controller.enqueue(remainingBytes); + } + } + } + /** * Handles the connection between the browser and the Arduino board via Web Serial. */ @@ -120,61 +157,42 @@ class SerialConnectionHandler { * If the timeout is reached, the reader will be canceled and the read lock will be released. */ async readBytes(numBytes, timeout = null) { - if (this.currentPort.readable.locked) { + if(!this.currentPort) return null; + if(this.currentPort.readable.locked) { console.log('🔒 Stream is already locked. Ignoring request...'); return null; } - const bytesRead = new Uint8Array(numBytes); - let bytesReadIdx = 0; - let keepReading = true; - - // As long as the errors are non-fatal, a new ReadableStream is created automatically and hence port.readable is non-null. - // If a fatal error occurs, such as the serial device being removed, then port.readable becomes null. - - while (this.currentPort?.readable && keepReading) { - const reader = this.currentPort.readable.getReader(); - this.currentReader = reader; - let timeoutID = null; - // let count = 0; - - try { - while (bytesReadIdx < numBytes) { - if (timeout) { - timeoutID = setTimeout(() => { - console.log('⌛️ Timeout occurred while reading.'); - if (this.currentPort?.readable) reader?.cancel(); - }, timeout); - } - - const { value, done } = await reader.read(); - if (timeoutID) clearTimeout(timeoutID); - - if (value) { - for (const byte of value) { - bytesRead[bytesReadIdx++] = byte; - if (bytesReadIdx >= numBytes) break; - } - // count += value.byteLength; - // console.log(`Read ${value.byteLength} (Total: ${count}) out of ${numBytes} bytes.}`); - } - - if (done) { - console.log('🚫 Reader has been canceled'); - break; - } - } - - } catch (error) { - console.error('💣 Error occurred while reading: ' + error.message); - } finally { - keepReading = false; - // console.log('🔓 Releasing reader lock...'); - reader?.releaseLock(); - this.currentReader = null; + const transformer = new BytesWaitTransformer(numBytes); + const transformStream = new TransformStream(transformer); + const pipedStream = this.currentPort.readable.pipeThrough(transformStream); + const reader = pipedStream.getReader(); + this.currentReader = reader; + let timeoutID = null; + + try { + if (timeout) { + timeoutID = setTimeout(() => { + console.log('⌛️ Timeout occurred while reading.'); + if (this.currentPort?.readable) reader?.cancel(); + }, timeout); + } + const { value, done } = await reader.read(); + if (timeoutID) clearTimeout(timeoutID); + + if (done) { + console.log('🚫 Reader has been canceled'); + return null; } + return value; + } catch (error) { + console.error('💣 Error occurred while reading: ' + error.message); + } finally { + // console.log('🔓 Releasing reader lock...'); + await reader?.cancel(); // Discards any enqueued data + reader?.releaseLock(); + this.currentReader = null; } - return bytesRead; } async sendData(byteArray) { From 084393b15f5f522829325eb2978aab3af1baba5c Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Dec 2023 11:46:27 +0100 Subject: [PATCH 19/37] Fix deadlock --- .../extras/WebSerialCamera/imageDataProcessor.js | 3 +++ .../WebSerialCamera/serialConnectionHandler.js | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js index ad1a8fdf0..18c394207 100644 --- a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js +++ b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js @@ -1,3 +1,6 @@ +// This could be turned into a transform stream. +// See example here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js + class ImageDataProcessor { pixelFormatInfo = { "RGB565": { diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index 552fe35c6..e28727136 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -52,6 +52,7 @@ class SerialConnectionHandler { this.timeout = timeout; this.currentPort = null; this.currentReader = null; + this.readableStreamClosed = null; this.registerEvents(); } @@ -118,6 +119,7 @@ class SerialConnectionHandler { const port = this.currentPort; this.currentPort = null; await this.currentReader?.cancel(); + await this.readableStreamClosed.catch(() => { }); // Ignores the error await port.close(); console.log('🔌 Disconnected from serial port.'); if(this.onDisconnect) this.onDisconnect(); @@ -163,10 +165,12 @@ class SerialConnectionHandler { return null; } - const transformer = new BytesWaitTransformer(numBytes); - const transformStream = new TransformStream(transformer); - const pipedStream = this.currentPort.readable.pipeThrough(transformStream); - const reader = pipedStream.getReader(); + const transformStream = new TransformStream(new BytesWaitTransformer(numBytes)); + // pipeThrough() cannot be used because we need a promise that resolves when the stream is closed + // to be able to close the port. pipeTo() returns such a promise. + // SEE: https://stackoverflow.com/questions/71262432/how-can-i-close-a-web-serial-port-that-ive-piped-through-a-transformstream + this.readableStreamClosed = this.currentPort.readable.pipeTo(transformStream.writable); + const reader = transformStream.readable.getReader(); this.currentReader = reader; let timeoutID = null; @@ -190,6 +194,7 @@ class SerialConnectionHandler { } finally { // console.log('🔓 Releasing reader lock...'); await reader?.cancel(); // Discards any enqueued data + await this.readableStreamClosed.catch(() => { }); // Ignores the error reader?.releaseLock(); this.currentReader = null; } From 302c8a6108f980fe81dc07021d239b987865e4ae Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Dec 2023 12:04:50 +0100 Subject: [PATCH 20/37] Add documentation to image processor --- .../WebSerialCamera/imageDataProcessor.js | 79 +++++++++++++++++-- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js index 18c394207..d471191ff 100644 --- a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js +++ b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js @@ -1,6 +1,8 @@ -// This could be turned into a transform stream. -// See example here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js - +/** + * Represents an image data processor that converts raw image data to a specified pixel format. + * This could be turned into a transform stream and be used in the serial connection handler. + * See example here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js + */ class ImageDataProcessor { pixelFormatInfo = { "RGB565": { @@ -16,11 +18,19 @@ class ImageDataProcessor { "bytesPerPixel": 3 }, "BAYER": { - "convert": null, // TODO + "convert": () => {throw new Error("BAYER conversion not implemented.")}, "bytesPerPixel": 1 } }; + /** + * Creates a new instance of the imageDataProcessor class. + * @param {CanvasRenderingContext2D} context - The 2D rendering context of the canvas. + * @param {string|null} mode - The image mode of the image data processor. (Optional) + * Possible values: RGB565, GRAYSCALE, RGB888, BAYER + * @param {number|null} width - The width of the image data processor. (Optional) + * @param {number|null} height - The height of the image data processor. (Optional) + */ constructor(context, mode = null, width = null, height = null) { this.context = context; this.canvas = context.canvas; @@ -29,24 +39,50 @@ class ImageDataProcessor { if(width && height) this.setResolution(width, height); } + /** + * Sets the image mode of the image data processor. + * Possible values: RGB565, GRAYSCALE, RGB888, BAYER + * + * @param {string} mode - The image mode of the image data processor. + */ setMode(mode) { this.mode = mode; this.bytesPerPixel = this.pixelFormatInfo[mode].bytesPerPixel; } + /** + * Sets the resolution of the target image. + * @param {number} width - The width of the resolution. + * @param {number} height - The height of the resolution. + */ setResolution(width, height) { this.width = width; this.height = height; } + /** + * Calculates the total number of bytes in the image data + * based on the current image mode and resolution. + * + * @returns {number} The total number of bytes. + */ getTotalBytes() { return this.width * this.height * this.bytesPerPixel; } + /** + * Checks if the image data processor is configured. + * This is true if the image mode and resolution are set. + * @returns {boolean} True if the image data processor is configured, false otherwise. + */ isConfigured() { return this.mode && this.width && this.height; } + /** + * Resets the state of the imageDataProcessor. + * This resets the image mode, resolution, and bytes per pixel. + */ reset() { this.mode = null; this.bytesPerPixel = null; @@ -54,6 +90,11 @@ class ImageDataProcessor { this.height = null; } + /** + * Converts a pixel value from RGB565 format to RGB888 format. + * @param {number} pixelValue - The pixel value in RGB565 format. + * @returns {number[]} - The RGB888 pixel value as an array of three values [R, G, B]. + */ convertRGB565ToRGB888(pixelValue) { // RGB565 let r = (pixelValue >> (6 + 5)) & 0x1F; @@ -66,16 +107,32 @@ class ImageDataProcessor { return [r, g, b]; } + /** + * Converts a grayscale pixel value to RGB888 format. + * @param {number} pixelValue - The grayscale pixel value. + * @returns {number[]} - The RGB888 pixel value as an array of three values [R, G, B]. + */ convertGrayScaleToRGB888(pixelValue) { return [pixelValue, pixelValue, pixelValue]; } + /** + * Converts a pixel value to RGB888 format. + * @param {number} pixelValue - The pixel value to convert. + * @returns {number[]} - The RGB888 pixel value as an array of three values [R, G, B]. + */ convertToRGB888(pixelValue){ - return [pixelValue[0], pixelValue[1], pixelValue[2]]; + return pixelValue; } - - // Get the pixel value using big endian - // Big-endian: the most significant byte comes first + + /** + * Retrieves the pixel value from the source data at the specified index + * using big endian: the most significant byte comes first. + * + * @param {Uint8Array} sourceData - The source data array. + * @param {number} index - The index of the pixel value in the source data array. + * @returns {number} The pixel value. + */ getPixelValue(sourceData, index) { if (this.bytesPerPixel == 1) { return sourceData[index]; @@ -90,6 +147,12 @@ class ImageDataProcessor { return 0; } + /** + * Retrieves the image data from the given bytes by converting each pixel value. + * + * @param {Uint8Array} bytes - The raw byte array containing the image data. + * @returns {ImageData} The image data object. + */ getImageData(bytes) { const BYTES_PER_ROW = this.width * this.bytesPerPixel; From 8d7a3fab07dae575a8abd977a810c8326f13db53 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Dec 2023 12:10:48 +0100 Subject: [PATCH 21/37] Add docs to config file --- .../Camera/extras/WebSerialCamera/cameraConfig.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libraries/Camera/extras/WebSerialCamera/cameraConfig.js b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js index d3a31286b..f068b94d8 100644 --- a/libraries/Camera/extras/WebSerialCamera/cameraConfig.js +++ b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js @@ -1,9 +1,21 @@ +/** + * @fileoverview This file contains the configuration for the camera. + */ + +/** + * The available camera (color) modes. + * The Arduino sketch uses the same values to communicate which mode should be used. + **/ const CAMERA_MODES = { 0: "GRAYSCALE", 1: "BAYER", 2: "RGB565" }; +/** + * The available camera resolutions. + * The Arduino sketch uses the same values to communicate which resolution should be used. + */ const CAMERA_RESOLUTIONS = { 0: { "name": "QQVGA", From 1e5cdc8f0e0238bf60d201d8a831bed6f12058aa Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Mon, 8 Jan 2024 15:20:20 +0100 Subject: [PATCH 22/37] Use default serial values --- libraries/Camera/extras/WebSerialCamera/app.js | 17 +++++------------ .../WebSerialCamera/serialConnectionHandler.js | 3 ++- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 89497ebb4..8a5884413 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -9,16 +9,9 @@ const ctx = canvas.getContext('2d'); // https://developer.chrome.com/articles/serial/ // https://wicg.github.io/serial/ -// Set the buffer size to the total bytes. This allows to read the entire bitmap in one go. -const bufferSize = 2 * 1024 * 1024; // Max buffer size is 16MB -const flowControl = 'hardware'; -const baudRate = 115200; // Adjust this value based on your device's baud rate -const dataBits = 8; // Adjust this value based on your device's data bits -const stopBits = 2; // Adjust this value based on your device's stop bits -const parityBit = 'even'; // Adjust this value based on your device's parity bit const imageDataProcessor = new ImageDataProcessor(ctx); -const connectionHandler = new SerialConnectionHandler(baudRate, dataBits, stopBits, parityBit, flowControl, bufferSize); +const connectionHandler = new SerialConnectionHandler(); connectionHandler.onConnect = async () => { connectButton.textContent = 'Disconnect'; @@ -43,9 +36,9 @@ connectionHandler.onDisconnect = () => { imageDataProcessor.reset(); }; -function renderBitmap(imageData) { - canvas.width = imageDataProcessor.width; - canvas.height = imageDataProcessor.height; +function renderBitmap(width, height, imageData) { + canvas.width = width; + canvas.height = height; ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.putImageData(imageData, 0, 0); } @@ -62,7 +55,7 @@ async function renderFrame(){ if(!bytes || bytes.length == 0) return false; // Nothing to render // console.log(`Reading done ✅. Rendering image...`); const imageData = imageDataProcessor.getImageData(bytes); - renderBitmap(imageData); + renderBitmap(imageDataProcessor.width, imageDataProcessor.height, imageData); return true; } diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index e28727136..1c86d9bb2 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -42,11 +42,12 @@ class BytesWaitTransformer { * Handles the connection between the browser and the Arduino board via Web Serial. */ class SerialConnectionHandler { - constructor(baudRate = 115200, dataBits = 8, stopBits = 1, parity = "none", flowControl = "none", bufferSize = 4096, timeout = 2000) { + constructor(baudRate = 115200, dataBits = 8, stopBits = 1, parity = "none", flowControl = "none", bufferSize = 2 * 1024 * 1024, timeout = 2000) { this.baudRate = baudRate; this.dataBits = dataBits; this.stopBits = stopBits; this.flowControl = flowControl; + // Max buffer size is 16MB this.bufferSize = bufferSize; this.parity = parity; this.timeout = timeout; From 8a3ca3b6cb40210cbc34ed4ecd718d96366446a3 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Mon, 8 Jan 2024 16:19:12 +0100 Subject: [PATCH 23/37] Remove dependency on 2D Context --- .../extras/WebSerialCamera/imageDataProcessor.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js index d471191ff..d40ad7d0c 100644 --- a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js +++ b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js @@ -2,6 +2,8 @@ * Represents an image data processor that converts raw image data to a specified pixel format. * This could be turned into a transform stream and be used in the serial connection handler. * See example here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js + * + * @author Sebastian Romero */ class ImageDataProcessor { pixelFormatInfo = { @@ -31,10 +33,7 @@ class ImageDataProcessor { * @param {number|null} width - The width of the image data processor. (Optional) * @param {number|null} height - The height of the image data processor. (Optional) */ - constructor(context, mode = null, width = null, height = null) { - this.context = context; - this.canvas = context.canvas; - + constructor(mode = null, width = null, height = null) { if(mode) this.setMode(mode); if(width && height) this.setResolution(width, height); } @@ -151,13 +150,11 @@ class ImageDataProcessor { * Retrieves the image data from the given bytes by converting each pixel value. * * @param {Uint8Array} bytes - The raw byte array containing the image data. - * @returns {ImageData} The image data object. + * @returns {Uint8ClampedArray} The image data as a Uint8ClampedArray containing RGBA values. */ getImageData(bytes) { const BYTES_PER_ROW = this.width * this.bytesPerPixel; - - const imageData = this.context.createImageData(this.width, this.height); - const dataContainer = imageData.data; + const dataContainer = new Uint8ClampedArray(this.width * this.height * 4); // 4 channels: R, G, B, A for (let row = 0; row < this.height; row++) { for (let col = 0; col < this.width; col++) { @@ -172,6 +169,6 @@ class ImageDataProcessor { dataContainer[pixelIndex + 3] = 255; // Alpha channel (opacity) } } - return imageData; + return dataContainer; } } \ No newline at end of file From 29adcd896535f750df520f5d06d8e4c06dcfb77d Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Mon, 8 Jan 2024 16:21:20 +0100 Subject: [PATCH 24/37] Working with new transformer --- .../CameraCaptureRawBytes.ino | 2 +- .../Camera/extras/WebSerialCamera/app.js | 16 +++- .../extras/WebSerialCamera/cameraConfig.js | 1 + .../Camera/extras/WebSerialCamera/index.html | 1 + .../serialConnectionHandler.js | 53 ++++-------- .../extras/WebSerialCamera/transformers.js | 84 +++++++++++++++++++ 6 files changed, 115 insertions(+), 42 deletions(-) create mode 100644 libraries/Camera/extras/WebSerialCamera/transformers.js diff --git a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino index 3cd988a70..6a27da9d1 100644 --- a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino +++ b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino @@ -96,7 +96,7 @@ void sendCameraConfig(){ void loop() { if(!Serial) { - Serial.begin(115200, SERIAL_8E2); + Serial.begin(115200); while(!Serial); } diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 8a5884413..e3d360fac 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -1,3 +1,8 @@ +/** + * @fileoverview This file contains the main application logic. + * @author Sebastian Romero + */ + const connectButton = document.getElementById('connect'); const refreshButton = document.getElementById('refresh'); const startButton = document.getElementById('start'); @@ -10,7 +15,8 @@ const ctx = canvas.getContext('2d'); // https://wicg.github.io/serial/ -const imageDataProcessor = new ImageDataProcessor(ctx); +const imageDataProcessor = new ImageDataProcessor(); +let imageDataTransfomer = new ImageDataTransformer(); const connectionHandler = new SerialConnectionHandler(); connectionHandler.onConnect = async () => { @@ -28,6 +34,9 @@ connectionHandler.onConnect = async () => { } imageDataProcessor.setMode(imageMode); imageDataProcessor.setResolution(imageResolution.width, imageResolution.height); + imageDataTransfomer.setImageMode(imageMode); + imageDataTransfomer.setResolution(imageResolution.width, imageResolution.height); + connectionHandler.setTransformer(imageDataTransfomer); renderStream(); }; @@ -54,7 +63,10 @@ async function renderFrame(){ const bytes = await connectionHandler.getFrame(imageDataProcessor.getTotalBytes()); if(!bytes || bytes.length == 0) return false; // Nothing to render // console.log(`Reading done ✅. Rendering image...`); - const imageData = imageDataProcessor.getImageData(bytes); + const imageData = ctx.createImageData(320, 240); + const data = imageDataProcessor.getImageData(bytes); + imageData.data.set(data); + renderBitmap(imageDataProcessor.width, imageDataProcessor.height, imageData); return true; } diff --git a/libraries/Camera/extras/WebSerialCamera/cameraConfig.js b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js index f068b94d8..1c0474473 100644 --- a/libraries/Camera/extras/WebSerialCamera/cameraConfig.js +++ b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js @@ -1,5 +1,6 @@ /** * @fileoverview This file contains the configuration for the camera. + * @author Sebastian Romero */ /** diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index 91a5c1147..7e26f5e8d 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -16,6 +16,7 @@ + diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index 1c86d9bb2..f05cb1048 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -1,43 +1,6 @@ const ArduinoUSBVendorId = 0x2341; const UserActionAbortError = 8; -class BytesWaitTransformer { - constructor(waitBytes) { - this.waitBytes = waitBytes; - this.buffer = new Uint8Array(0); - this.controller = undefined; - } - - async transform(chunk, controller) { - this.controller = controller; - - // Concatenate incoming chunk with existing buffer - this.buffer = new Uint8Array([...this.buffer, ...chunk]); - - while (this.buffer.length >= this.waitBytes) { - // Extract the required number of bytes - const bytesToProcess = this.buffer.slice(0, this.waitBytes); - - // Remove processed bytes from the buffer - this.buffer = this.buffer.slice(this.waitBytes); - - // Notify the controller that bytes have been processed - controller.enqueue(bytesToProcess); - } - } - - flush(controller) { - if (this.buffer.length > 0) { - // Handle remaining bytes (if any) when the stream is closed - const remainingBytes = this.buffer.slice(); - console.log("Remaining bytes:", remainingBytes); - - // Notify the controller that remaining bytes have been processed - controller.enqueue(remainingBytes); - } - } - } - /** * Handles the connection between the browser and the Arduino board via Web Serial. */ @@ -54,6 +17,7 @@ class SerialConnectionHandler { this.currentPort = null; this.currentReader = null; this.readableStreamClosed = null; + this.transformer = new BytesWaitTransformer(); this.registerEvents(); } @@ -74,6 +38,14 @@ class SerialConnectionHandler { } } + /** + * Sets the transformer that is used to convert bytes into higher-level data types. + * @param {*} transformer + */ + setTransformer(transformer) { + this.transformer = transformer; + } + /** * Checks if the browser is connected to a serial port. * @returns {boolean} True if the browser is connected, false otherwise. @@ -121,6 +93,7 @@ class SerialConnectionHandler { this.currentPort = null; await this.currentReader?.cancel(); await this.readableStreamClosed.catch(() => { }); // Ignores the error + this.transformer.flush(); await port.close(); console.log('🔌 Disconnected from serial port.'); if(this.onDisconnect) this.onDisconnect(); @@ -165,8 +138,9 @@ class SerialConnectionHandler { console.log('🔒 Stream is already locked. Ignoring request...'); return null; } - - const transformStream = new TransformStream(new BytesWaitTransformer(numBytes)); + + this.transformer.setBytesToWait(numBytes); + const transformStream = new TransformStream(this.transformer); // pipeThrough() cannot be used because we need a promise that resolves when the stream is closed // to be able to close the port. pipeTo() returns such a promise. // SEE: https://stackoverflow.com/questions/71262432/how-can-i-close-a-web-serial-port-that-ive-piped-through-a-transformstream @@ -180,6 +154,7 @@ class SerialConnectionHandler { timeoutID = setTimeout(() => { console.log('⌛️ Timeout occurred while reading.'); if (this.currentPort?.readable) reader?.cancel(); + this.transformer.flush(); }, timeout); } const { value, done } = await reader.read(); diff --git a/libraries/Camera/extras/WebSerialCamera/transformers.js b/libraries/Camera/extras/WebSerialCamera/transformers.js new file mode 100644 index 000000000..6794521d8 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/transformers.js @@ -0,0 +1,84 @@ +class BytesWaitTransformer { + constructor(waitBytes = 1) { + this.waitBytes = waitBytes; + this.buffer = new Uint8Array(0); + this.controller = undefined; + } + + setBytesToWait(waitBytes) { + this.waitBytes = waitBytes; + } + + /** + * Converts bytes into higher-level data types. + * This method is meant to be overridden by subclasses. + * @param {Uint8Array} bytes + * @returns + */ + convertBytes(bytes) { + return bytes; + } + + + async transform(chunk, controller) { + this.controller = controller; + + // Concatenate incoming chunk with existing buffer + this.buffer = new Uint8Array([...this.buffer, ...chunk]); + + while (this.buffer.length >= this.waitBytes) { + // Extract the required number of bytes + const bytesToProcess = this.buffer.slice(0, this.waitBytes); + + // Remove processed bytes from the buffer + this.buffer = this.buffer.slice(this.waitBytes); + + // Notify the controller that bytes have been processed + controller.enqueue(this.convertBytes(bytesToProcess)); + } + } + + flush(controller) { + if (this.buffer.length > 0) { + // Handle remaining bytes (if any) when the stream is closed + const remainingBytes = this.buffer.slice(); + console.log("Remaining bytes:", remainingBytes); + + // Notify the controller that remaining bytes have been processed + controller.enqueue(remainingBytes); + } + } +} + +class ImageDataTransformer extends BytesWaitTransformer { + constructor(context, width, height, imageMode) { + super(1); + this.width = width; + this.height = height; + } + + setResolution(width, height) { + this.width = width; + this.height = height; + } + + setImageMode(imageMode) { + this.imageMode = imageMode; + } + + convertBytes(bytes) { + console.log("Converting bytes"); + let a = new Uint8Array(bytes); + // Iterate over UInt8Array + for (let i = 0; i < a.length; i++) { + a[i] = a[i] * 2; + } + + // const imageData = new ImageData(this.width, this.height); + // for (let i = 0; i < bytes.length; i++) { + // imageData.data[i] = bytes[i]; + // } + // return imageData; + return bytes; + } +} \ No newline at end of file From d7d081940b6812f4f6731f73b851079e2661d0a3 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Mon, 8 Jan 2024 17:26:25 +0100 Subject: [PATCH 25/37] Add documentation to app file --- .../Camera/extras/WebSerialCamera/app.js | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index e3d360fac..42257cd6d 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -1,5 +1,19 @@ /** * @fileoverview This file contains the main application logic. + * + * The application uses the Web Serial API to connect to the serial port. + * Check the following links for more information on the Web Serial API: + * https://developer.chrome.com/articles/serial/ + * https://wicg.github.io/serial/ + * + * The flow of the application is as follows: + * 1. The user clicks the "Connect" button or the browser automatically connects + * to the serial port if it has been previously connected. + * 2. The application requests the camera configuration (mode and resolution) from the board. + * 3. The application starts reading the image data stream from the serial port. + * It waits until the calculated number of bytes have been read and then processes the data. + * 4. The processed image data is rendered on the canvas. + * * @author Sebastian Romero */ @@ -10,14 +24,11 @@ const saveImageButton = document.getElementById('save-image'); const canvas = document.getElementById('bitmapCanvas'); const ctx = canvas.getContext('2d'); -// Check the following links for more information on the Web Serial API: -// https://developer.chrome.com/articles/serial/ -// https://wicg.github.io/serial/ +const imageDataTransfomer = new ImageDataTransformer(ctx); +const connectionHandler = new SerialConnectionHandler(); -const imageDataProcessor = new ImageDataProcessor(); -let imageDataTransfomer = new ImageDataTransformer(); -const connectionHandler = new SerialConnectionHandler(); +// Connection handler event listeners connectionHandler.onConnect = async () => { connectButton.textContent = 'Disconnect'; @@ -32,46 +43,55 @@ connectionHandler.onConnect = async () => { console.error(`🚫 Invalid camera configuration: ${cameraConfig[0]}, ${cameraConfig[1]}. Aborting...`); return; } - imageDataProcessor.setMode(imageMode); - imageDataProcessor.setResolution(imageResolution.width, imageResolution.height); imageDataTransfomer.setImageMode(imageMode); imageDataTransfomer.setResolution(imageResolution.width, imageResolution.height); - connectionHandler.setTransformer(imageDataTransfomer); renderStream(); }; connectionHandler.onDisconnect = () => { connectButton.textContent = 'Connect'; - imageDataProcessor.reset(); + imageDataTransfomer.reset(); }; -function renderBitmap(width, height, imageData) { - canvas.width = width; - canvas.height = height; - ctx.clearRect(0, 0, canvas.width, canvas.height); - ctx.putImageData(imageData, 0, 0); -} + +// Rendering logic async function renderStream(){ while(connectionHandler.isConnected()){ - if(imageDataProcessor.isConfigured()) await renderFrame(); + if(imageDataTransfomer.isConfigured()) await renderFrame(); } } +/** + * Renders the image data for one frame from the board and renders it. + * @returns {Promise} True if a frame was rendered, false otherwise. + */ async function renderFrame(){ if(!connectionHandler.isConnected()) return; - const bytes = await connectionHandler.getFrame(imageDataProcessor.getTotalBytes()); - if(!bytes || bytes.length == 0) return false; // Nothing to render - // console.log(`Reading done ✅. Rendering image...`); - const imageData = ctx.createImageData(320, 240); - const data = imageDataProcessor.getImageData(bytes); - imageData.data.set(data); - - renderBitmap(imageDataProcessor.width, imageDataProcessor.height, imageData); + const imageData = await connectionHandler.getFrame(imageDataTransfomer); + if(!imageData) return false; // Nothing to render + if(!(imageData instanceof ImageData)) throw new Error('🚫 Image data is not of type ImageData'); + renderBitmap(ctx, imageData); return true; } +/** + * Renders the image data on the canvas. + * @param {CanvasRenderingContext2D} context The canvas context to render on. + * @param {ImageData} imageData The image data to render. + */ +function renderBitmap(context, imageData) { + context.canvas.width = imageData.width; + context.canvas.height = imageData.height; + context.clearRect(0, 0, canvas.width, canvas.height); + context.putImageData(imageData, 0, 0); +} + + +// UI Event listeners + startButton.addEventListener('click', renderStream); + connectButton.addEventListener('click', async () => { if(connectionHandler.isConnected()){ connectionHandler.disconnectSerial(); @@ -80,8 +100,9 @@ connectButton.addEventListener('click', async () => { await connectionHandler.connectSerial(); } }); + refreshButton.addEventListener('click', () => { - if(imageDataProcessor.isConfigured()) renderFrame(); + if(imageDataTransfomer.isConfigured()) renderFrame(); }); saveImageButton.addEventListener('click', () => { From c6567937479a048d74b6df12849fede52e739a3e Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Mon, 8 Jan 2024 18:17:46 +0100 Subject: [PATCH 26/37] Add more documentation --- .../WebSerialCamera/imageDataProcessor.js | 18 +-- .../serialConnectionHandler.js | 80 +++++++++---- .../extras/WebSerialCamera/transformers.js | 111 +++++++++++++++--- 3 files changed, 151 insertions(+), 58 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js index d40ad7d0c..83ff2a650 100644 --- a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js +++ b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js @@ -1,7 +1,5 @@ /** * Represents an image data processor that converts raw image data to a specified pixel format. - * This could be turned into a transform stream and be used in the serial connection handler. - * See example here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js * * @author Sebastian Romero */ @@ -27,14 +25,13 @@ class ImageDataProcessor { /** * Creates a new instance of the imageDataProcessor class. - * @param {CanvasRenderingContext2D} context - The 2D rendering context of the canvas. * @param {string|null} mode - The image mode of the image data processor. (Optional) * Possible values: RGB565, GRAYSCALE, RGB888, BAYER * @param {number|null} width - The width of the image data processor. (Optional) * @param {number|null} height - The height of the image data processor. (Optional) */ constructor(mode = null, width = null, height = null) { - if(mode) this.setMode(mode); + if(mode) this.setImageMode(mode); if(width && height) this.setResolution(width, height); } @@ -44,7 +41,7 @@ class ImageDataProcessor { * * @param {string} mode - The image mode of the image data processor. */ - setMode(mode) { + setImageMode(mode) { this.mode = mode; this.bytesPerPixel = this.pixelFormatInfo[mode].bytesPerPixel; } @@ -69,15 +66,6 @@ class ImageDataProcessor { return this.width * this.height * this.bytesPerPixel; } - /** - * Checks if the image data processor is configured. - * This is true if the image mode and resolution are set. - * @returns {boolean} True if the image data processor is configured, false otherwise. - */ - isConfigured() { - return this.mode && this.width && this.height; - } - /** * Resets the state of the imageDataProcessor. * This resets the image mode, resolution, and bytes per pixel. @@ -152,7 +140,7 @@ class ImageDataProcessor { * @param {Uint8Array} bytes - The raw byte array containing the image data. * @returns {Uint8ClampedArray} The image data as a Uint8ClampedArray containing RGBA values. */ - getImageData(bytes) { + convertToPixelData(bytes) { const BYTES_PER_ROW = this.width * this.bytesPerPixel; const dataContainer = new Uint8ClampedArray(this.width * this.height * 4); // 4 channels: R, G, B, A diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index f05cb1048..599500863 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -1,23 +1,40 @@ +/** + * @fileoverview This file contains the SerialConnectionHandler class. + * It handles the connection between the browser and the Arduino board via Web Serial. + * @author Sebastian Romero + */ + const ArduinoUSBVendorId = 0x2341; const UserActionAbortError = 8; /** * Handles the connection between the browser and the Arduino board via Web Serial. + * Please note that for board with software serial over USB, the baud rate and other serial settings have no effect. */ class SerialConnectionHandler { + /** + * Represents a serial connection handler. + * @constructor + * @param {number} [baudRate=115200] - The baud rate of the serial connection. + * @param {number} [dataBits=8] - The number of data bits. + * @param {number} [stopBits=1] - The number of stop bits. + * @param {string} [parity="none"] - The parity setting. + * @param {string} [flowControl="none"] - The flow control setting. + * @param {number} [bufferSize=2097152] - The size of the buffer in bytes. Max buffer size is 16MB + * @param {number} [timeout=2000] - The connection timeout value in milliseconds. + */ constructor(baudRate = 115200, dataBits = 8, stopBits = 1, parity = "none", flowControl = "none", bufferSize = 2 * 1024 * 1024, timeout = 2000) { this.baudRate = baudRate; this.dataBits = dataBits; this.stopBits = stopBits; this.flowControl = flowControl; - // Max buffer size is 16MB this.bufferSize = bufferSize; this.parity = parity; this.timeout = timeout; this.currentPort = null; this.currentReader = null; + this.currentTransformer = null; this.readableStreamClosed = null; - this.transformer = new BytesWaitTransformer(); this.registerEvents(); } @@ -38,14 +55,6 @@ class SerialConnectionHandler { } } - /** - * Sets the transformer that is used to convert bytes into higher-level data types. - * @param {*} transformer - */ - setTransformer(transformer) { - this.transformer = transformer; - } - /** * Checks if the browser is connected to a serial port. * @returns {boolean} True if the browser is connected, false otherwise. @@ -93,7 +102,7 @@ class SerialConnectionHandler { this.currentPort = null; await this.currentReader?.cancel(); await this.readableStreamClosed.catch(() => { }); // Ignores the error - this.transformer.flush(); + this.currentTransformer?.flush(); await port.close(); console.log('🔌 Disconnected from serial port.'); if(this.onDisconnect) this.onDisconnect(); @@ -126,21 +135,31 @@ class SerialConnectionHandler { return false; } + + /** + * Reads a specified number of bytes from the serial connection. + * @param {number} numBytes - The number of bytes to read. + * @returns {Promise} - A promise that resolves to a Uint8Array containing the read bytes. + */ + async readBytes(numBytes) { + return await this.readData(new BytesWaitTransformer(numBytes)); + } + /** * Reads the specified number of bytes from the serial port. - * @param {number} numBytes The number of bytes to read. - * @param {number} timeout The timeout in milliseconds. + * @param {Transformer} transformer The transformer that is used to process the bytes. * If the timeout is reached, the reader will be canceled and the read lock will be released. */ - async readBytes(numBytes, timeout = null) { + async readData(transformer) { + if(!transformer) throw new Error('Transformer is null'); if(!this.currentPort) return null; if(this.currentPort.readable.locked) { console.log('🔒 Stream is already locked. Ignoring request...'); return null; } - this.transformer.setBytesToWait(numBytes); - const transformStream = new TransformStream(this.transformer); + const transformStream = new TransformStream(transformer); + this.currentTransformer = transformer; // pipeThrough() cannot be used because we need a promise that resolves when the stream is closed // to be able to close the port. pipeTo() returns such a promise. // SEE: https://stackoverflow.com/questions/71262432/how-can-i-close-a-web-serial-port-that-ive-piped-through-a-transformstream @@ -150,12 +169,12 @@ class SerialConnectionHandler { let timeoutID = null; try { - if (timeout) { + if (this.timeout) { timeoutID = setTimeout(() => { console.log('⌛️ Timeout occurred while reading.'); if (this.currentPort?.readable) reader?.cancel(); this.transformer.flush(); - }, timeout); + }, this.timeout); } const { value, done } = await reader.read(); if (timeoutID) clearTimeout(timeoutID); @@ -173,9 +192,16 @@ class SerialConnectionHandler { await this.readableStreamClosed.catch(() => { }); // Ignores the error reader?.releaseLock(); this.currentReader = null; + this.currentTransformer = null; } } + /** + * Sends the provided byte array data through the current serial port. + * + * @param {ArrayBuffer} byteArray - The byte array data to send. + * @returns {Promise} - A promise that resolves when the data has been sent. + */ async sendData(byteArray) { if (!this.currentPort?.writable) { console.log('🚫 Port is not writable. Ignoring request...'); @@ -196,10 +222,19 @@ class SerialConnectionHandler { return this.sendData([1]); } + /** + * Requests the camera configuration from the board by writing a 2 to the serial port. + * @returns {Promise} A promise that resolves with the configuration data. + */ async requestConfig() { return this.sendData([2]); } + /** + * Requests the camera resolution from the board and reads it back from the serial port. + * The configuration simply consists of two bytes: the mode and the resolution. + * @returns {Promise} The raw configuration data as an ArrayBuffer. + */ async getConfig() { if (!this.currentPort) return; @@ -211,15 +246,12 @@ class SerialConnectionHandler { /** * Requests a frame from the Arduino board and reads the specified number of bytes from the serial port afterwards. * Times out after the timeout in milliseconds specified in the constructor. - * @param {number} totalBytes The number of bytes to read. + * @param {Transformer} transformer The transformer that is used to process the bytes. */ - async getFrame(totalBytes) { + async getFrame(transformer) { if (!this.currentPort) return; - await this.requestFrame(); - // console.log(`Trying to read ${totalBytes} bytes...`); - // Read the given amount of bytes - return await this.readBytes(totalBytes, this.timeout); + return await this.readData(transformer, this.timeout); } /** diff --git a/libraries/Camera/extras/WebSerialCamera/transformers.js b/libraries/Camera/extras/WebSerialCamera/transformers.js index 6794521d8..2ff89dbcb 100644 --- a/libraries/Camera/extras/WebSerialCamera/transformers.js +++ b/libraries/Camera/extras/WebSerialCamera/transformers.js @@ -1,3 +1,6 @@ +/** + * A transformer class that waits for a specific number of bytes before processing them. + */ class BytesWaitTransformer { constructor(waitBytes = 1) { this.waitBytes = waitBytes; @@ -5,6 +8,10 @@ class BytesWaitTransformer { this.controller = undefined; } + /** + * Sets the number of bytes to wait before processing the data. + * @param {number} waitBytes - The number of bytes to wait. + */ setBytesToWait(waitBytes) { this.waitBytes = waitBytes; } @@ -19,7 +26,13 @@ class BytesWaitTransformer { return bytes; } - + /** + * Transforms the incoming chunk of data and enqueues the processed bytes to the controller. + * It does so when the buffer contains at least the specified number of bytes. + * @param {Uint8Array} chunk - The incoming chunk of data. + * @param {TransformStreamDefaultController} controller - The controller for enqueuing processed bytes. + * @returns {Promise} - A promise that resolves when the transformation is complete. + */ async transform(chunk, controller) { this.controller = controller; @@ -38,6 +51,11 @@ class BytesWaitTransformer { } } + /** + * Flushes the buffer and processes any remaining bytes when the stream is closed. + * + * @param {WritableStreamDefaultController} controller - The controller for the writable stream. + */ flush(controller) { if (this.buffer.length > 0) { // Handle remaining bytes (if any) when the stream is closed @@ -45,40 +63,95 @@ class BytesWaitTransformer { console.log("Remaining bytes:", remainingBytes); // Notify the controller that remaining bytes have been processed - controller.enqueue(remainingBytes); + controller?.enqueue(remainingBytes); } } } + +/** + * Represents an Image Data Transformer that converts bytes into image data. + * See other example for PNGs here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js + * @extends BytesWaitTransformer + */ class ImageDataTransformer extends BytesWaitTransformer { - constructor(context, width, height, imageMode) { - super(1); - this.width = width; - this.height = height; + /** + * Creates a new instance of the Transformer class. + * @param {CanvasRenderingContext2D} context - The canvas rendering context. + * @param {number} [width=null] - The width of the image. + * @param {number} [height=null] - The height of the image. + * @param {string} [imageMode=null] - The image mode. + */ + constructor(context, width = null, height = null, imageMode = null) { + super(); + this.context = context; + this.imageDataProcessor = new ImageDataProcessor(); + if (width && height){ + this.setResolution(width, height); + } + if (imageMode){ + this.setImageMode(imageMode); + } } + /** + * Sets the resolution of the camera image that is being processed. + * + * @param {number} width - The width of the resolution. + * @param {number} height - The height of the resolution. + */ setResolution(width, height) { this.width = width; this.height = height; + this.imageDataProcessor.setResolution(width, height); + if(this.isConfigured()){ + this.setBytesToWait(this.imageDataProcessor.getTotalBytes()); + } } + /** + * Sets the image mode of the camera image that is being processed. + * Possible values: RGB565, GRAYSCALE, RGB888, BAYER + * + * @param {string} imageMode - The image mode to set. + */ setImageMode(imageMode) { this.imageMode = imageMode; + this.imageDataProcessor.setImageMode(imageMode); + if(this.isConfigured()){ + this.setBytesToWait(this.imageDataProcessor.getTotalBytes()); + } } - convertBytes(bytes) { - console.log("Converting bytes"); - let a = new Uint8Array(bytes); - // Iterate over UInt8Array - for (let i = 0; i < a.length; i++) { - a[i] = a[i] * 2; - } + /** + * Checks if the image data processor is configured. + * This is true if the image mode and resolution are set. + * @returns {boolean} True if the image data processor is configured, false otherwise. + */ + isConfigured() { + return this.imageMode && this.width && this.height; + } - // const imageData = new ImageData(this.width, this.height); - // for (let i = 0; i < bytes.length; i++) { - // imageData.data[i] = bytes[i]; - // } - // return imageData; - return bytes; + /** + * Resets the state of the transformer. + */ + reset() { + this.imageMode = null; + this.width = null; + this.height = null; + this.imageDataProcessor.reset(); + } + + /** + * Converts the given raw bytes into an ImageData object by using the ImageDataProcessor. + * + * @param {Uint8Array} bytes - The bytes to convert. + * @returns {ImageData} The converted ImageData object. + */ + convertBytes(bytes) { + const pixelData = this.imageDataProcessor.convertToPixelData(bytes); + const imageData = this.context.createImageData(imageDataTransfomer.width, imageDataTransfomer.height); + imageData.data.set(pixelData); + return imageData; } } \ No newline at end of file From 28d4e371ef74384329e486207feac4adab9203c7 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 9 Jan 2024 12:27:30 +0100 Subject: [PATCH 27/37] Add dedicated sketch for WebSerial --- .../CameraCaptureWebSerial.ino | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino diff --git a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino new file mode 100644 index 000000000..e3d3c5f5c --- /dev/null +++ b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino @@ -0,0 +1,127 @@ +/* + * This example shows how to capture images from the camera and send them over Web Serial. + * + * There is a companion web app that receives the images and displays them in a canvas. + * It can be found in the "extras" folder of this library. + * The on-board LED lights up while the image is being sent over serial. + * + * Instructions: + * 1. Make sure the correct camera is selected in the #include section below by uncommenting the correct line. + * 2. Upload this sketch to your camera-equipped board. + * 3. Open the web app in a browser (Chrome or Edge) by opening the index.html file in the "extras" folder. + * + * Initial author: Sebastian Romero @sebromero + */ + +#include "camera.h" + +#ifdef ARDUINO_NICLA_VISION + #include "gc2145.h" + GC2145 galaxyCore; + Camera cam(galaxyCore); + #define IMAGE_MODE CAMERA_RGB565 +#elif defined(ARDUINO_PORTENTA_H7_M7) + // uncomment the correct camera in use + #include "hm0360.h" + HM0360 himax; + // #include "himax.h"; + // HM01B0 himax; + Camera cam(himax); + #define IMAGE_MODE CAMERA_GRAYSCALE +#elif defined(ARDUINO_GIGA) + #include "ov767x.h" + // uncomment the correct camera in use + OV7670 ov767x; + // OV7675 ov767x; + Camera cam(ov767x); + #define IMAGE_MODE CAMERA_RGB565 +#else +#error "This board is unsupported." +#endif + +/* +Other buffer instantiation options: + FrameBuffer fb(0x30000000); + FrameBuffer fb(320,240,2); + +If resolution higher than 320x240 is required, please use external RAM via + #include "SDRAM.h" + FrameBuffer fb(SDRAM_START_ADDRESS); + ... + // and adding in setup() + SDRAM.begin(); +*/ +#define CHUNK_SIZE 512 // Size of chunks in bytes +#define RESOLUTION CAMERA_R320x240 +FrameBuffer fb; + +unsigned long lastUpdate = 0; + +void blinkLED(uint32_t count = 0xFFFFFFFF) { + while (count--) { + digitalWrite(LED_BUILTIN, LOW); // turn the LED on (HIGH is the voltage level) + delay(50); // wait for a second + digitalWrite(LED_BUILTIN, HIGH); // turn the LED off by making the voltage LOW + delay(50); // wait for a second + } +} + +void setup() { + pinMode(LED_BUILTIN, OUTPUT); + + // Init the cam QVGA, 30FPS + if (!cam.begin(RESOLUTION, IMAGE_MODE, 30)) { + blinkLED(); + } + + blinkLED(5); +} + +void sendFrame(){ + // Grab frame and write to serial + if (cam.grabFrame(fb, 3000) == 0) { + byte* buffer = fb.getBuffer(); + size_t bufferSize = cam.frameSize(); + digitalWrite(LED_BUILTIN, LOW); + + // Split buffer into chunks + for(size_t i = 0; i < bufferSize; i += CHUNK_SIZE) { + size_t chunkSize = min(bufferSize - i, CHUNK_SIZE); + Serial.write(buffer + i, chunkSize); + Serial.flush(); + delay(1); // Optional: Add a small delay to allow the receiver to process the chunk + } + + digitalWrite(LED_BUILTIN, HIGH); + } else { + blinkLED(20); + } +} + +void sendCameraConfig(){ + Serial.write(IMAGE_MODE); + Serial.write(RESOLUTION); + Serial.flush(); + delay(1); +} + +void loop() { + if(!Serial) { + Serial.begin(115200); + while(!Serial); + } + + if(!Serial.available()) return; + + byte request = Serial.read(); + + switch(request){ + case 1: + sendFrame(); + break; + case 2: + sendCameraConfig(); + break; + } + +} From b6d20c444da76c0eb99d151dcf33ce3a3feeb0ea Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 9 Jan 2024 12:27:43 +0100 Subject: [PATCH 28/37] Add filters --- .../Camera/extras/WebSerialCamera/app.js | 6 + .../Camera/extras/WebSerialCamera/filters.js | 207 ++++++++++++++++++ .../Camera/extras/WebSerialCamera/index.html | 1 + .../extras/WebSerialCamera/transformers.js | 14 +- 4 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 libraries/Camera/extras/WebSerialCamera/filters.js diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 42257cd6d..5890856a6 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -25,6 +25,12 @@ const canvas = document.getElementById('bitmapCanvas'); const ctx = canvas.getContext('2d'); const imageDataTransfomer = new ImageDataTransformer(ctx); +// 🐣 Uncomment one of the following lines to apply a filter to the image data +// imageDataTransfomer.filter = new GrayScaleFilter(); +// imageDataTransfomer.filter = new BlackAndWhiteFilter(); +// imageDataTransfomer.filter = new SepiaColorFilter(); +// imageDataTransfomer.filter = new PixelateFilter(8); +// imageDataTransfomer.filter = new BlurFilter(8); const connectionHandler = new SerialConnectionHandler(); diff --git a/libraries/Camera/extras/WebSerialCamera/filters.js b/libraries/Camera/extras/WebSerialCamera/filters.js new file mode 100644 index 000000000..0b7622898 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/filters.js @@ -0,0 +1,207 @@ +/** + * @fileoverview This file contains the filters that can be applied to an image. + * @author Sebastian Romero + */ + +/** + * Represents an image filter interface. This class is meant to be extended by subclasses. + */ +class ImageFilter { +/** + * Applies a filter to the given pixel data. + * @param {Uint8Array} pixelData - The pixel data to apply the filter to. The pixel data gets modified in place. + * @param {number} [width=null] - The width of the image. Defaults to null. + * @param {number} [height=null] - The height of the image. Defaults to null. + * @throws {Error} - Throws an error if the applyFilter method is not implemented. + */ + applyFilter(pixelData, width = null, height = null) { + throw new Error('applyFilter not implemented'); + } +} + +/** + * Represents a grayscale filter that converts an image to grayscale. + * @extends ImageFilter + */ +class GrayScaleFilter extends ImageFilter { + /** + * Applies the grayscale filter to the given pixel data. + * @param {Uint8ClampedArray} pixelData - The pixel data to apply the filter to. + * @param {number} [width=null] - The width of the image. + * @param {number} [height=null] - The height of the image. + */ + applyFilter(pixelData, width = null, height = null) { + for (let i = 0; i < pixelData.length; i += 4) { + const r = pixelData[i]; + const g = pixelData[i + 1]; + const b = pixelData[i + 2]; + const gray = (r + g + b) / 3; + pixelData[i] = gray; + pixelData[i + 1] = gray; + pixelData[i + 2] = gray; + } + } +} + +/** + * A class representing a black and white image filter. + * @extends ImageFilter + */ +class BlackAndWhiteFilter extends ImageFilter { + applyFilter(pixelData, width = null, height = null) { + for (let i = 0; i < pixelData.length; i += 4) { + const r = pixelData[i]; + const g = pixelData[i + 1]; + const b = pixelData[i + 2]; + const gray = (r + g + b) / 3; + const bw = gray > 127 ? 255 : 0; + pixelData[i] = bw; + pixelData[i + 1] = bw; + pixelData[i + 2] = bw; + } + } +} + +/** + * Represents a color filter that applies a sepia tone effect to an image. + * @extends ImageFilter + */ +class SepiaColorFilter extends ImageFilter { + applyFilter(pixelData, width = null, height = null) { + for (let i = 0; i < pixelData.length; i += 4) { + const r = pixelData[i]; + const g = pixelData[i + 1]; + const b = pixelData[i + 2]; + const gray = (r + g + b) / 3; + pixelData[i] = gray + 100; + pixelData[i + 1] = gray + 50; + pixelData[i + 2] = gray; + } + } +} + +/** + * Represents a filter that applies a pixelation effect to an image. + * @extends ImageFilter + */ +class PixelateFilter extends ImageFilter { + + constructor(blockSize = 8){ + super(); + this.blockSize = blockSize; + } + + applyFilter(pixelData, width, height) { + for (let y = 0; y < height; y += this.blockSize) { + for (let x = 0; x < width; x += this.blockSize) { + const blockAverage = this.getBlockAverage(x, y, width, height, pixelData, this.blockSize); + + // Set all pixels in the block to the calculated average color + for (let blockY = 0; blockY < this.blockSize && y + blockY < height; blockY++) { + for (let blockX = 0; blockX < this.blockSize && x + blockX < width; blockX++) { + const pixelIndex = ((y + blockY) * width + (x + blockX)) * 4; + pixelData[pixelIndex] = blockAverage.red; + pixelData[pixelIndex + 1] = blockAverage.green; + pixelData[pixelIndex + 2] = blockAverage.blue; + } + } + } + } + } + + /** + * Calculates the average RGB values of a block of pixels. + * + * @param {number} x - The x-coordinate of the top-left corner of the block. + * @param {number} y - The y-coordinate of the top-left corner of the block. + * @param {number} width - The width of the image. + * @param {number} height - The height of the image. + * @param {Uint8ClampedArray} pixels - The array of pixel data. + * @returns {Object} - An object containing the average red, green, and blue values. + */ + getBlockAverage(x, y, width, height, pixels) { + let totalRed = 0; + let totalGreen = 0; + let totalBlue = 0; + const blockSizeSquared = this.blockSize * this.blockSize; + + for (let blockY = 0; blockY < this.blockSize && y + blockY < height; blockY++) { + for (let blockX = 0; blockX < this.blockSize && x + blockX < width; blockX++) { + const pixelIndex = ((y + blockY) * width + (x + blockX)) * 4; + totalRed += pixels[pixelIndex]; + totalGreen += pixels[pixelIndex + 1]; + totalBlue += pixels[pixelIndex + 2]; + } + } + + return { + red: totalRed / blockSizeSquared, + green: totalGreen / blockSizeSquared, + blue: totalBlue / blockSizeSquared, + }; + } + +} + +/** + * Represents a filter that applies a blur effect to an image. + * @extends ImageFilter + */ +class BlurFilter extends ImageFilter { + constructor(radius = 8) { + super(); + this.radius = radius; + } + + applyFilter(pixelData, width, height) { + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const pixelIndex = (y * width + x) * 4; + + const averageColor = this.getAverageColor(x, y, width, height, pixelData, this.radius); + pixelData[pixelIndex] = averageColor.red; + pixelData[pixelIndex + 1] = averageColor.green; + pixelData[pixelIndex + 2] = averageColor.blue; + } + } + } + + /** + * Calculates the average color of a rectangular region in an image. + * + * @param {number} x - The x-coordinate of the top-left corner of the region. + * @param {number} y - The y-coordinate of the top-left corner of the region. + * @param {number} width - The width of the region. + * @param {number} height - The height of the region. + * @param {Uint8ClampedArray} pixels - The pixel data of the image. + * @param {number} radius - The radius of the neighborhood to consider for each pixel. + * @returns {object} - An object representing the average color of the region, with red, green, and blue components. + */ + getAverageColor(x, y, width, height, pixels, radius) { + let totalRed = 0; + let totalGreen = 0; + let totalBlue = 0; + let pixelCount = 0; + + for (let offsetY = -radius; offsetY <= radius; offsetY++) { + for (let offsetX = -radius; offsetX <= radius; offsetX++) { + const neighborX = x + offsetX; + const neighborY = y + offsetY; + + if (neighborX >= 0 && neighborX < width && neighborY >= 0 && neighborY < height) { + const pixelIndex = (neighborY * width + neighborX) * 4; + totalRed += pixels[pixelIndex]; + totalGreen += pixels[pixelIndex + 1]; + totalBlue += pixels[pixelIndex + 2]; + pixelCount++; + } + } + } + + return { + red: totalRed / pixelCount, + green: totalGreen / pixelCount, + blue: totalBlue / pixelCount, + }; + } +} diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html index 7e26f5e8d..3f677a2ff 100644 --- a/libraries/Camera/extras/WebSerialCamera/index.html +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -16,6 +16,7 @@ + diff --git a/libraries/Camera/extras/WebSerialCamera/transformers.js b/libraries/Camera/extras/WebSerialCamera/transformers.js index 2ff89dbcb..60bccdb88 100644 --- a/libraries/Camera/extras/WebSerialCamera/transformers.js +++ b/libraries/Camera/extras/WebSerialCamera/transformers.js @@ -1,3 +1,9 @@ +/** + * @fileoverview This file contains classes that transform incoming data into higher-level data types. + * @author Sebastian Romero + */ + + /** * A transformer class that waits for a specific number of bytes before processing them. */ @@ -68,7 +74,6 @@ class BytesWaitTransformer { } } - /** * Represents an Image Data Transformer that converts bytes into image data. * See other example for PNGs here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js @@ -149,7 +154,12 @@ class ImageDataTransformer extends BytesWaitTransformer { * @returns {ImageData} The converted ImageData object. */ convertBytes(bytes) { - const pixelData = this.imageDataProcessor.convertToPixelData(bytes); + let pixelData = this.imageDataProcessor.convertToPixelData(bytes); + + if(this.filter){ + this.filter.applyFilter(pixelData, imageDataTransfomer.width, imageDataTransfomer.height); + } + const imageData = this.context.createImageData(imageDataTransfomer.width, imageDataTransfomer.height); imageData.data.set(pixelData); return imageData; From 41fbc7a3f15be923875719714a403d329fc6cc0a Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 9 Jan 2024 12:27:53 +0100 Subject: [PATCH 29/37] Fix incorrect variable name --- .../Camera/extras/WebSerialCamera/serialConnectionHandler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index 599500863..8776718be 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -173,7 +173,7 @@ class SerialConnectionHandler { timeoutID = setTimeout(() => { console.log('⌛️ Timeout occurred while reading.'); if (this.currentPort?.readable) reader?.cancel(); - this.transformer.flush(); + this.currentTransformer.flush(); }, this.timeout); } const { value, done } = await reader.read(); From e343588e3e906cacb9f6d7f3f19af007c4b158f9 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 9 Jan 2024 12:27:57 +0100 Subject: [PATCH 30/37] Add documentation --- libraries/Camera/extras/WebSerialCamera/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 libraries/Camera/extras/WebSerialCamera/README.md diff --git a/libraries/Camera/extras/WebSerialCamera/README.md b/libraries/Camera/extras/WebSerialCamera/README.md new file mode 100644 index 000000000..04888bbcc --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/README.md @@ -0,0 +1,12 @@ +# 📹 WebSerial Camera Stream + +This folder contains a web application that provides a camera stream over WebSerial. +This is an experimental feature not supported in all browsers. It's recommended to use Google Chrome. +See [Browser Compatibility](https://developer.mozilla.org/en-US/docs/Web/API/Web_Serial_API#browser_compatibility) + +## Instructions + +1. Upload the companion [Arduino sketch](../../examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino) to your board. +2. Open the web app either by directly opening the index.html file or serving it via a webserver and opening the URL provided by the webserver. +3. Click "Connect". Your board's serial port should show up in the popup. Select it. Click once again "Connect". The camera feed should start. If the board has been previously connected to the browser, it will connect automatically. +4. (Optional) click "Save Image" if you want to save individual camera frames to your computer. \ No newline at end of file From 4c58a74b656f56180e44ad1fd2c3b6aac45fd41b Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Tue, 9 Jan 2024 12:36:36 +0100 Subject: [PATCH 31/37] Restore original sketch --- .../CameraCaptureRawBytes.ino | 74 +++++++------------ 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino index 6a27da9d1..a0ca20775 100644 --- a/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino +++ b/libraries/Camera/examples/CameraCaptureRawBytes/CameraCaptureRawBytes.ino @@ -9,8 +9,11 @@ // uncomment the correct camera in use #include "hm0360.h" HM0360 himax; - // #include "himax.h"; + + // #include "himax.h" // HM01B0 himax; + // Camera cam(himax); + Camera cam(himax); #define IMAGE_MODE CAMERA_GRAYSCALE #elif defined(ARDUINO_GIGA) @@ -36,16 +39,14 @@ If resolution higher than 320x240 is required, please use external RAM via // and adding in setup() SDRAM.begin(); */ -#define CHUNK_SIZE 512 // Size of chunks in bytes -#define RESOLUTION CAMERA_R320x240 FrameBuffer fb; unsigned long lastUpdate = 0; + void blinkLED(uint32_t count = 0xFFFFFFFF) { - pinMode(LED_BUILTIN, OUTPUT); - + pinMode(LED_BUILTIN, OUTPUT); while (count--) { digitalWrite(LED_BUILTIN, LOW); // turn the LED on (HIGH is the voltage level) delay(50); // wait for a second @@ -56,61 +57,36 @@ void blinkLED(uint32_t count = 0xFFFFFFFF) void setup() { // Init the cam QVGA, 30FPS - if (!cam.begin(RESOLUTION, IMAGE_MODE, 30)) { + if (!cam.begin(CAMERA_R320x240, IMAGE_MODE, 30)) { blinkLED(); } blinkLED(5); - - pinMode(LEDB, OUTPUT); - digitalWrite(LEDB, HIGH); -} - -void sendFrame(){ - // Grab frame and write to serial - if (cam.grabFrame(fb, 3000) == 0) { - byte* buffer = fb.getBuffer(); - size_t bufferSize = cam.frameSize(); - digitalWrite(LEDB, LOW); - - // Split buffer into chunks - for(size_t i = 0; i < bufferSize; i += CHUNK_SIZE) { - size_t chunkSize = min(bufferSize - i, CHUNK_SIZE); - Serial.write(buffer + i, chunkSize); - Serial.flush(); - delay(1); // Optional: Add a small delay to allow the receiver to process the chunk - } - - digitalWrite(LEDB, HIGH); - } else { - blinkLED(20); - } -} - -void sendCameraConfig(){ - Serial.write(IMAGE_MODE); - Serial.write(RESOLUTION); - Serial.flush(); - delay(1); } void loop() { if(!Serial) { Serial.begin(115200); - while(!Serial); + while(!Serial); } - if(!Serial.available()) return; - - byte request = Serial.read(); - - switch(request){ - case 1: - sendFrame(); - break; - case 2: - sendCameraConfig(); - break; + // Time out after 2 seconds, which sets the (constant) frame rate + bool timeoutDetected = millis() - lastUpdate > 2000; + + // Wait for sync byte and timeout + // Notice that this order must be kept, or the sync bytes will be + // consumed prematurely + if ((!timeoutDetected) || (Serial.read() != 1)) + { + return; } + + lastUpdate = millis(); + // Grab frame and write to serial + if (cam.grabFrame(fb, 3000) == 0) { + Serial.write(fb.getBuffer(), cam.frameSize()); + } else { + blinkLED(20); + } } From e9aa024e50086875d1ef1008e77c46f37d733b2b Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 17 Jan 2024 06:58:50 -0300 Subject: [PATCH 32/37] Add setter for connection timeout --- .../WebSerialCamera/serialConnectionHandler.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js index 8776718be..6f23c644f 100644 --- a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -20,8 +20,8 @@ class SerialConnectionHandler { * @param {number} [stopBits=1] - The number of stop bits. * @param {string} [parity="none"] - The parity setting. * @param {string} [flowControl="none"] - The flow control setting. - * @param {number} [bufferSize=2097152] - The size of the buffer in bytes. Max buffer size is 16MB - * @param {number} [timeout=2000] - The connection timeout value in milliseconds. + * @param {number} [bufferSize=2097152] - The size of the buffer in bytes. The default value is 2 MB. Max buffer size is 16MB. + * @param {number} [timeout=2000] - The connection timeout value in milliseconds. The default value is 2000 ms. */ constructor(baudRate = 115200, dataBits = 8, stopBits = 1, parity = "none", flowControl = "none", bufferSize = 2 * 1024 * 1024, timeout = 2000) { this.baudRate = baudRate; @@ -38,6 +38,14 @@ class SerialConnectionHandler { this.registerEvents(); } + /** + * Sets the connection timeout for the serial connection. + * @param {number} timeout - The timeout value in milliseconds. + */ + setConnectionTimeout(timeout) { + this.timeout = timeout; + } + /** * Prompts the user to select a serial port. * @returns {Promise} The serial port that the user has selected. From c9299d45ebf57a8c9b293b963475dd07bac4f76c Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 17 Jan 2024 07:15:00 -0300 Subject: [PATCH 33/37] Add start stop sequence transformer --- .../CameraCaptureWebSerial.ino | 11 +- .../Camera/extras/WebSerialCamera/app.js | 5 +- .../extras/WebSerialCamera/transformers.js | 166 +++++++++++++++++- 3 files changed, 177 insertions(+), 5 deletions(-) diff --git a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino index e3d3c5f5c..a5e8bbd81 100644 --- a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino +++ b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino @@ -52,7 +52,9 @@ If resolution higher than 320x240 is required, please use external RAM via SDRAM.begin(); */ #define CHUNK_SIZE 512 // Size of chunks in bytes -#define RESOLUTION CAMERA_R320x240 +#define RESOLUTION CAMERA_R320x240 // CAMERA_R160x120 +constexpr uint8_t START_SEQUENCE[4] = { 0xfa, 0xce, 0xfe, 0xed }; +constexpr uint8_t STOP_SEQUENCE[4] = { 0xda, 0xbb, 0xad, 0x00 }; FrameBuffer fb; unsigned long lastUpdate = 0; @@ -84,6 +86,10 @@ void sendFrame(){ size_t bufferSize = cam.frameSize(); digitalWrite(LED_BUILTIN, LOW); + Serial.write(START_SEQUENCE, sizeof(START_SEQUENCE)); + Serial.flush(); + delay(1); + // Split buffer into chunks for(size_t i = 0; i < bufferSize; i += CHUNK_SIZE) { size_t chunkSize = min(bufferSize - i, CHUNK_SIZE); @@ -91,6 +97,9 @@ void sendFrame(){ Serial.flush(); delay(1); // Optional: Add a small delay to allow the receiver to process the chunk } + Serial.write(STOP_SEQUENCE, sizeof(STOP_SEQUENCE)); + Serial.flush(); + delay(1); digitalWrite(LED_BUILTIN, HIGH); } else { diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js index 5890856a6..68823b347 100644 --- a/libraries/Camera/extras/WebSerialCamera/app.js +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -11,7 +11,7 @@ * to the serial port if it has been previously connected. * 2. The application requests the camera configuration (mode and resolution) from the board. * 3. The application starts reading the image data stream from the serial port. - * It waits until the calculated number of bytes have been read and then processes the data. + * It waits until the expected amount of bytes have been read and then processes the data. * 4. The processed image data is rendered on the canvas. * * @author Sebastian Romero @@ -25,6 +25,9 @@ const canvas = document.getElementById('bitmapCanvas'); const ctx = canvas.getContext('2d'); const imageDataTransfomer = new ImageDataTransformer(ctx); +imageDataTransfomer.setStartSequence([0xfa, 0xce, 0xfe, 0xed]); +imageDataTransfomer.setStopSequence([0xda, 0xbb, 0xad, 0x00]); + // 🐣 Uncomment one of the following lines to apply a filter to the image data // imageDataTransfomer.filter = new GrayScaleFilter(); // imageDataTransfomer.filter = new BlackAndWhiteFilter(); diff --git a/libraries/Camera/extras/WebSerialCamera/transformers.js b/libraries/Camera/extras/WebSerialCamera/transformers.js index 60bccdb88..ef039afd2 100644 --- a/libraries/Camera/extras/WebSerialCamera/transformers.js +++ b/libraries/Camera/extras/WebSerialCamera/transformers.js @@ -4,6 +4,166 @@ */ +/** + * Represents a transformer that processes incoming data between start and stop sequences. + */ +class StartStopSequenceTransformer { + constructor(startSequence = null, stopSequence = null, expectedBytes = null) { + this.startSequence = new Uint8Array(startSequence); + this.stopSequence = new Uint8Array(stopSequence); + this.expectedBytes = expectedBytes; + this.buffer = new Uint8Array(0); + this.controller = undefined; + this.waitingForStart = true; + } + + /** + * Sets the start sequence for the received data. + * This is used to disregard any data before the start sequence. + * @param {Array} startSequence - The start sequence as an array of numbers. + */ + setStartSequence(startSequence) { + this.startSequence = new Uint8Array(startSequence); + } + + /** + * Sets the stop sequence for the received data. + * This is used to know when the data has finished being sent and should be processed. + * @param {Array} stopSequence - The stop sequence as an array of numbers. + */ + setStopSequence(stopSequence) { + this.stopSequence = new Uint8Array(stopSequence); + } + + /** + * Sets the expected number of bytes for the received data. + * This is used to check if the number of bytes matches the expected amount + * and discard the data if it doesn't. + * + * @param {number} expectedBytes - The expected number of bytes. + */ + setExpectedBytes(expectedBytes) { + this.expectedBytes = expectedBytes; + } + + /** + * Transforms the incoming chunk of data and enqueues the processed bytes to the controller + * between start and stop sequences. + * + * @param {Uint8Array} chunk - The incoming chunk of data. + * @param {TransformStreamDefaultController} controller - The controller for enqueuing processed bytes. + * @returns {Promise} - A promise that resolves when the transformation is complete. + */ + async transform(chunk, controller) { + this.controller = controller; + + // Concatenate incoming chunk with existing buffer + this.buffer = new Uint8Array([...this.buffer, ...chunk]); + + let startIndex = 0; + + while (startIndex < this.buffer.length) { + if (this.waitingForStart) { + // Look for the start sequence + startIndex = this.indexOfSequence(this.buffer, this.startSequence, startIndex); + + if (startIndex === -1) { + // No start sequence found, discard the buffer + this.buffer = new Uint8Array(0); + return; + } + + // Remove bytes before the start sequence + this.buffer = this.buffer.slice(startIndex + this.startSequence.length); + startIndex = 0; // Reset startIndex after removing bytes + this.waitingForStart = false; + } + + // Look for the stop sequence + const stopIndex = this.indexOfSequence(this.buffer, this.stopSequence, startIndex); + + if (stopIndex === -1) { + // No stop sequence found, wait for more data + return; + } + + // Extract bytes between start and stop sequences + const bytesToProcess = this.buffer.slice(startIndex, stopIndex); + this.buffer = this.buffer.slice(stopIndex + this.stopSequence.length); + + // Check if the number of bytes matches the expected amount + if (this.expectedBytes !== null && bytesToProcess.length !== this.expectedBytes) { + // Drop all bytes in the buffer to avoid broken data + throw new Error(`🚫 Expected ${this.expectedBytes} bytes, but got ${bytesToProcess.length} bytes instead.`); + this.buffer = new Uint8Array(0); + return; + } + + // Notify the controller that bytes have been processed + controller.enqueue(this.convertBytes(bytesToProcess)); + this.waitingForStart = true; + } + } + + /** + * Flushes the buffer and processes any remaining bytes when the stream is closed. + * + * @param {WritableStreamDefaultController} controller - The controller for the writable stream. + */ + flush(controller) { + // Only enqueue the remaining bytes if they meet the expectedBytes criteria + if (this.buffer.length === this.expectedBytes || this.expectedBytes === null) { + controller?.enqueue(this.buffer); + } + } + + /** + * Finds the index of the given sequence in the buffer. + * + * @param {Uint8Array} buffer - The buffer to search. + * @param {Uint8Array} sequence - The sequence to find. + * @param {number} startIndex - The index to start searching from. + * @returns {number} - The index of the sequence in the buffer, or -1 if not found. + */ + indexOfSequence(buffer, sequence, startIndex) { + for (let i = startIndex; i <= buffer.length - sequence.length; i++) { + if (this.isSubarray(buffer, sequence, i)) { + return i; + } + } + return -1; + } + + /** + * Checks if a subarray is present at a given index in the buffer. + * + * @param {Uint8Array} buffer - The buffer to check. + * @param {Uint8Array} subarray - The subarray to check. + * @param {number} index - The index to start checking from. + * @returns {boolean} - True if the subarray is present at the given index, false otherwise. + */ + isSubarray(buffer, subarray, index) { + for (let i = 0; i < subarray.length; i++) { + if (buffer[index + i] !== subarray[i]) { + return false; + } + } + return true; + } + + /** + * Converts bytes into higher-level data types. + * This method is meant to be overridden by subclasses. + * @param {Uint8Array} bytes + * @returns + */ + convertBytes(bytes) { + return bytes; + } + +} + + /** * A transformer class that waits for a specific number of bytes before processing them. */ @@ -77,9 +237,9 @@ class BytesWaitTransformer { /** * Represents an Image Data Transformer that converts bytes into image data. * See other example for PNGs here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js - * @extends BytesWaitTransformer + * @extends StartStopSequenceTransformer */ -class ImageDataTransformer extends BytesWaitTransformer { +class ImageDataTransformer extends StartStopSequenceTransformer { /** * Creates a new instance of the Transformer class. * @param {CanvasRenderingContext2D} context - The canvas rendering context. @@ -110,7 +270,7 @@ class ImageDataTransformer extends BytesWaitTransformer { this.height = height; this.imageDataProcessor.setResolution(width, height); if(this.isConfigured()){ - this.setBytesToWait(this.imageDataProcessor.getTotalBytes()); + this.setExpectedBytes(this.imageDataProcessor.getTotalBytes()); } } From 4f72204a31dd679b6c226a710742cec5f829530b Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 17 Jan 2024 12:56:49 -0300 Subject: [PATCH 34/37] Better handle data in transformer --- .../extras/WebSerialCamera/transformers.js | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/libraries/Camera/extras/WebSerialCamera/transformers.js b/libraries/Camera/extras/WebSerialCamera/transformers.js index ef039afd2..722353c20 100644 --- a/libraries/Camera/extras/WebSerialCamera/transformers.js +++ b/libraries/Camera/extras/WebSerialCamera/transformers.js @@ -59,10 +59,12 @@ class StartStopSequenceTransformer { // Concatenate incoming chunk with existing buffer this.buffer = new Uint8Array([...this.buffer, ...chunk]); - let startIndex = 0; - while (startIndex < this.buffer.length) { + // Only process data if at least one start and stop sequence is present in the buffer + const minimumRequiredBytes = Math.min(this.startSequence.length, this.stopSequence.length); + + while (this.buffer.length >= minimumRequiredBytes) { if (this.waitingForStart) { // Look for the start sequence startIndex = this.indexOfSequence(this.buffer, this.startSequence, startIndex); @@ -73,7 +75,7 @@ class StartStopSequenceTransformer { return; } - // Remove bytes before the start sequence + // Remove bytes before the start sequence including the start sequence this.buffer = this.buffer.slice(startIndex + this.startSequence.length); startIndex = 0; // Reset startIndex after removing bytes this.waitingForStart = false; @@ -89,13 +91,14 @@ class StartStopSequenceTransformer { // Extract bytes between start and stop sequences const bytesToProcess = this.buffer.slice(startIndex, stopIndex); + // Remove processed bytes from the buffer including the stop sequence. this.buffer = this.buffer.slice(stopIndex + this.stopSequence.length); // Check if the number of bytes matches the expected amount if (this.expectedBytes !== null && bytesToProcess.length !== this.expectedBytes) { - // Drop all bytes in the buffer to avoid broken data - throw new Error(`🚫 Expected ${this.expectedBytes} bytes, but got ${bytesToProcess.length} bytes instead.`); - this.buffer = new Uint8Array(0); + // Skip processing the bytes, but keep the remaining data in the buffer + console.error(`🚫 Expected ${this.expectedBytes} bytes, but got ${bytesToProcess.length} bytes instead. Dropping data.`); + this.waitingForStart = true; return; } @@ -106,17 +109,16 @@ class StartStopSequenceTransformer { } /** - * Flushes the buffer and processes any remaining bytes when the stream is closed. + * Flushes the buffer and discards any remaining bytes when the stream is closed. * * @param {WritableStreamDefaultController} controller - The controller for the writable stream. */ flush(controller) { - // Only enqueue the remaining bytes if they meet the expectedBytes criteria - if (this.buffer.length === this.expectedBytes || this.expectedBytes === null) { - controller?.enqueue(this.buffer); - } + // Discard the remaining data in the buffer + this.buffer = new Uint8Array(0); } + /** * Finds the index of the given sequence in the buffer. * From 5aa18d754ec129e9c83aa663d244627e819482cf Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 17 Jan 2024 12:57:12 -0300 Subject: [PATCH 35/37] Refactor example sketch --- .../CameraCaptureWebSerial.ino | 54 +++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino index a5e8bbd81..af84871c7 100644 --- a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino +++ b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino @@ -51,14 +51,20 @@ If resolution higher than 320x240 is required, please use external RAM via // and adding in setup() SDRAM.begin(); */ -#define CHUNK_SIZE 512 // Size of chunks in bytes -#define RESOLUTION CAMERA_R320x240 // CAMERA_R160x120 -constexpr uint8_t START_SEQUENCE[4] = { 0xfa, 0xce, 0xfe, 0xed }; -constexpr uint8_t STOP_SEQUENCE[4] = { 0xda, 0xbb, 0xad, 0x00 }; -FrameBuffer fb; +constexpr uint16_t CHUNK_SIZE = 512; // Size of chunks in bytes +constexpr uint8_t RESOLUTION = CAMERA_R320x240; // CAMERA_R160x120 +constexpr uint8_t CONFIG_SEND_REQUEST = 2; +constexpr uint8_t IMAGE_SEND_REQUEST = 1; -unsigned long lastUpdate = 0; +uint8_t START_SEQUENCE[4] = { 0xfa, 0xce, 0xfe, 0xed }; +uint8_t STOP_SEQUENCE[4] = { 0xda, 0xbb, 0xad, 0x00 }; +FrameBuffer fb; +/** + * Blinks the LED a specified number of times. + * + * @param count The number of times to blink the LED. Default is 0xFFFFFFFF. + */ void blinkLED(uint32_t count = 0xFFFFFFFF) { while (count--) { digitalWrite(LED_BUILTIN, LOW); // turn the LED on (HIGH is the voltage level) @@ -79,6 +85,21 @@ void setup() { blinkLED(5); } +/** + * Sends a chunk of data over a serial connection. + * + * @param buffer The buffer containing the data to be sent. + * @param bufferSize The size of the buffer. + */ +void sendChunk(uint8_t* buffer, size_t bufferSize){ + Serial.write(buffer, bufferSize); + Serial.flush(); + delay(1); // Optional: Add a small delay to allow the receiver to process the chunk +} + +/** + * Sends a frame of camera image data over a serial connection. + */ void sendFrame(){ // Grab frame and write to serial if (cam.grabFrame(fb, 3000) == 0) { @@ -86,20 +107,15 @@ void sendFrame(){ size_t bufferSize = cam.frameSize(); digitalWrite(LED_BUILTIN, LOW); - Serial.write(START_SEQUENCE, sizeof(START_SEQUENCE)); - Serial.flush(); - delay(1); + sendChunk(START_SEQUENCE, sizeof(START_SEQUENCE)); // Split buffer into chunks for(size_t i = 0; i < bufferSize; i += CHUNK_SIZE) { size_t chunkSize = min(bufferSize - i, CHUNK_SIZE); - Serial.write(buffer + i, chunkSize); - Serial.flush(); - delay(1); // Optional: Add a small delay to allow the receiver to process the chunk + sendChunk(buffer + i, chunkSize); } - Serial.write(STOP_SEQUENCE, sizeof(STOP_SEQUENCE)); - Serial.flush(); - delay(1); + + sendChunk(STOP_SEQUENCE, sizeof(STOP_SEQUENCE)); digitalWrite(LED_BUILTIN, HIGH); } else { @@ -107,6 +123,10 @@ void sendFrame(){ } } +/** + * Sends the camera configuration over a serial connection. + * This is used to configure the web app to display the image correctly. + */ void sendCameraConfig(){ Serial.write(IMAGE_MODE); Serial.write(RESOLUTION); @@ -125,10 +145,10 @@ void loop() { byte request = Serial.read(); switch(request){ - case 1: + case IMAGE_SEND_REQUEST: sendFrame(); break; - case 2: + case CONFIG_SEND_REQUEST: sendCameraConfig(); break; } From a6ea68f0b0b71cfb596e5883de0b543006390d43 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 17 Jan 2024 15:57:54 -0300 Subject: [PATCH 36/37] Use red LED to indicate errors --- .../CameraCaptureWebSerial.ino | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino index af84871c7..d5e69248a 100644 --- a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino +++ b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino @@ -62,27 +62,30 @@ FrameBuffer fb; /** * Blinks the LED a specified number of times. - * + * @param ledPin The pin number of the LED. * @param count The number of times to blink the LED. Default is 0xFFFFFFFF. */ -void blinkLED(uint32_t count = 0xFFFFFFFF) { +void blinkLED(int ledPin, uint32_t count = 0xFFFFFFFF) { while (count--) { - digitalWrite(LED_BUILTIN, LOW); // turn the LED on (HIGH is the voltage level) + digitalWrite(ledPin, LOW); // turn the LED on (HIGH is the voltage level) delay(50); // wait for a second - digitalWrite(LED_BUILTIN, HIGH); // turn the LED off by making the voltage LOW + digitalWrite(ledPin, HIGH); // turn the LED off by making the voltage LOW delay(50); // wait for a second } } void setup() { pinMode(LED_BUILTIN, OUTPUT); + pinMode(LEDR, OUTPUT); + digitalWrite(LED_BUILTIN, HIGH); + digitalWrite(LEDR, HIGH); // Init the cam QVGA, 30FPS if (!cam.begin(RESOLUTION, IMAGE_MODE, 30)) { - blinkLED(); + blinkLED(LEDR); } - blinkLED(5); + blinkLED(LED_BUILTIN, 5); } /** From b696b5934fb1bcd5311f262c98de9882f5ead93f Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Wed, 17 Jan 2024 22:29:54 -0300 Subject: [PATCH 37/37] Clarify sketch instructions --- .../examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino index d5e69248a..8347f725a 100644 --- a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino +++ b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino @@ -8,7 +8,8 @@ * Instructions: * 1. Make sure the correct camera is selected in the #include section below by uncommenting the correct line. * 2. Upload this sketch to your camera-equipped board. - * 3. Open the web app in a browser (Chrome or Edge) by opening the index.html file in the "extras" folder. + * 3. Open the web app in a browser (Chrome or Edge) by opening the index.html file + * in the "WebSerialCamera" folder which is located in the "extras" folder. * * Initial author: Sebastian Romero @sebromero */