From dcaed3291f28c9ced02938d428da1da1eeeb4068 Mon Sep 17 00:00:00 2001 From: dominictootell Date: Sat, 22 Dec 2018 13:15:33 +0000 Subject: [PATCH 1/6] Expose Text Encoder for use in workers --- lib/__tests__/cloudworker-e2e.test.js | 18 ++++++++++++++++++ lib/runtime.js | 5 +++++ lib/runtime/text-encoder.js | 4 ++++ 3 files changed, 27 insertions(+) create mode 100644 lib/runtime/text-encoder.js diff --git a/lib/__tests__/cloudworker-e2e.test.js b/lib/__tests__/cloudworker-e2e.test.js index b541c73..a759179 100644 --- a/lib/__tests__/cloudworker-e2e.test.js +++ b/lib/__tests__/cloudworker-e2e.test.js @@ -211,4 +211,22 @@ describe('cloudworker-e2e', async () => { await upstream.close() cb() }) + + test('simple text encoder', async (cb) => { + const script = ` + addEventListener('fetch', event => { + const encoder = new TextEncoder() + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + writer.write(encoder.encode('hello')).then(() => writer.close()) + event.respondWith(new Response(readable, {status: 200})) + }) + ` + const server = new Cloudworker(script).listen(8080) + const res = await axios.get('http://localhost:8080', defaultAxiosOpts) + expect(res.status).toEqual(200) + expect(res.data).toEqual('hello') + await server.close() + cb() + }) }) diff --git a/lib/runtime.js b/lib/runtime.js index 44deaf0..14720cc 100644 --- a/lib/runtime.js +++ b/lib/runtime.js @@ -2,6 +2,7 @@ const { Request, Response, fetch, Headers, freezeHeaders } = require('./runtime/ const { URL } = require('./runtime/url') const { ReadableStream, WritableStream, TransformStream } = require('./runtime/stream') const { FetchEvent } = require('./runtime/fetch-event') +const { TextDecoder, TextEncoder } = require('./runtime/text-encoder') class Context { constructor (addEventListener, cacheFactory, bindings = {}) { @@ -18,6 +19,8 @@ class Context { this.TransformStream = TransformStream this.FetchEvent = FetchEvent this.caches = cacheFactory + this.TextDecoder = TextDecoder + this.TextEncoder = TextEncoder } } @@ -33,4 +36,6 @@ module.exports = { WritableStream, TransformStream, URL, + TextDecoder, + TextEncoder, } diff --git a/lib/runtime/text-encoder.js b/lib/runtime/text-encoder.js new file mode 100644 index 0000000..1530fb4 --- /dev/null +++ b/lib/runtime/text-encoder.js @@ -0,0 +1,4 @@ +const textEncoding = require('text-encoding') + +module.exports.TextDecoder = textEncoding.TextDecoder +module.exports.TextEncoder = textEncoding.TextEncoder From b87391632d3bbeb47575036eb19ed54d396dea98 Mon Sep 17 00:00:00 2001 From: dominictootell Date: Sun, 23 Dec 2018 00:00:19 +0000 Subject: [PATCH 2/6] TextDecoder implementation as text-encoding's wasn't working --- lib/__tests__/cloudworker-e2e.test.js | 19 ++++++++++++ lib/runtime/text-encoder.js | 43 ++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/lib/__tests__/cloudworker-e2e.test.js b/lib/__tests__/cloudworker-e2e.test.js index a759179..2098746 100644 --- a/lib/__tests__/cloudworker-e2e.test.js +++ b/lib/__tests__/cloudworker-e2e.test.js @@ -229,4 +229,23 @@ describe('cloudworker-e2e', async () => { await server.close() cb() }) + + test('simple text encoder and decoder', async (cb) => { + const script = ` + addEventListener('fetch', event => { + const helloBytes = new Uint8Array([104, 101, 108, 108, 111]) + const decoder = new TextDecoder() + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + writer.write(new TextEncoder().encode(decoder.decode(helloBytes))).then(() => writer.close()) + event.respondWith(new Response(readable, {status: 200})) + }) + ` + const server = new Cloudworker(script).listen(8080) + const res = await axios.get('http://localhost:8080', defaultAxiosOpts) + expect(res.status).toEqual(200) + expect(res.data).toEqual('hello') + await server.close() + cb() + }) }) diff --git a/lib/runtime/text-encoder.js b/lib/runtime/text-encoder.js index 1530fb4..b19cb13 100644 --- a/lib/runtime/text-encoder.js +++ b/lib/runtime/text-encoder.js @@ -1,4 +1,45 @@ const textEncoding = require('text-encoding') -module.exports.TextDecoder = textEncoding.TextDecoder +module.exports.TextDecoder = TextDecoder module.exports.TextEncoder = textEncoding.TextEncoder + +// code from https://github.com/golang/go/issues/27295 +function TextDecoder () { +} + +TextDecoder.prototype.decode = (octets) => { + var string = '' + var i = 0 + while (i < octets.length) { + var octet = octets[i] + var bytesNeeded = 0 + var codePoint = 0 + if (octet <= 0x7F) { + bytesNeeded = 0 + codePoint = octet & 0xFF + } else if (octet <= 0xDF) { + bytesNeeded = 1 + codePoint = octet & 0x1F + } else if (octet <= 0xEF) { + bytesNeeded = 2 + codePoint = octet & 0x0F + } else if (octet <= 0xF4) { + bytesNeeded = 3 + codePoint = octet & 0x07 + } + if (octets.length - i - bytesNeeded > 0) { + var k = 0 + while (k < bytesNeeded) { + octet = octets[i + k + 1] + codePoint = (codePoint << 6) | (octet & 0x3F) + k += 1 + } + } else { + codePoint = 0xFFFD + bytesNeeded = octets.length - i + } + string += String.fromCodePoint(codePoint) + i += bytesNeeded + 1 + } + return string +} \ No newline at end of file From 2b38e3560f8b78919eabed91ccd910ea685dc4bc Mon Sep 17 00:00:00 2001 From: dominictootell Date: Thu, 3 Jan 2019 22:52:27 +0000 Subject: [PATCH 3/6] text-encoder's decode was not indentifying the buffer as an ArrayBuffer The result being the decode method ended up in creating an empty bytes view : https://github.com/inexorabletash/text-encoding/blob/master/lib/encoding.js#L1086. The CustomTextDecoder here ensures the Uint8Array's buffer is an ArrayBuffer, resulting in TextDecoders method setting up bytes view correctly --- lib/runtime.js | 3 +- lib/runtime/text-encoder.js | 60 +++++++++++++------------------------ 2 files changed, 21 insertions(+), 42 deletions(-) diff --git a/lib/runtime.js b/lib/runtime.js index 14720cc..9693aef 100644 --- a/lib/runtime.js +++ b/lib/runtime.js @@ -36,6 +36,5 @@ module.exports = { WritableStream, TransformStream, URL, - TextDecoder, - TextEncoder, + } diff --git a/lib/runtime/text-encoder.js b/lib/runtime/text-encoder.js index b19cb13..b241c3e 100644 --- a/lib/runtime/text-encoder.js +++ b/lib/runtime/text-encoder.js @@ -1,45 +1,25 @@ const textEncoding = require('text-encoding') -module.exports.TextDecoder = TextDecoder -module.exports.TextEncoder = textEncoding.TextEncoder - -// code from https://github.com/golang/go/issues/27295 -function TextDecoder () { +function CustomTextDecoder () { + if (!(this instanceof CustomTextDecoder)) { + throw TypeError('Called as a function. Did you forget \'new\'?') + } + this.decoder = new textEncoding.TextDecoder() + return this } -TextDecoder.prototype.decode = (octets) => { - var string = '' - var i = 0 - while (i < octets.length) { - var octet = octets[i] - var bytesNeeded = 0 - var codePoint = 0 - if (octet <= 0x7F) { - bytesNeeded = 0 - codePoint = octet & 0xFF - } else if (octet <= 0xDF) { - bytesNeeded = 1 - codePoint = octet & 0x1F - } else if (octet <= 0xEF) { - bytesNeeded = 2 - codePoint = octet & 0x0F - } else if (octet <= 0xF4) { - bytesNeeded = 3 - codePoint = octet & 0x07 - } - if (octets.length - i - bytesNeeded > 0) { - var k = 0 - while (k < bytesNeeded) { - octet = octets[i + k + 1] - codePoint = (codePoint << 6) | (octet & 0x3F) - k += 1 - } - } else { - codePoint = 0xFFFD - bytesNeeded = octets.length - i - } - string += String.fromCodePoint(codePoint) - i += bytesNeeded + 1 +CustomTextDecoder.prototype.decode = function decode (input, options) { + if (Object.prototype.toString.call(input) === '[object Uint8Array]') { + var buffer = new ArrayBuffer(input.length) + var view = new Uint8Array(buffer) + view.set(input) + return this.decoder.decode( + view, + options) } - return string -} \ No newline at end of file + + return this.decoder.decode(input, options) +} + +module.exports.TextDecoder = CustomTextDecoder +module.exports.TextEncoder = textEncoding.TextEncoder From 0763e04e813023bb1be0f5e84b0a228ea8493a2c Mon Sep 17 00:00:00 2001 From: dominictootell Date: Sun, 6 Jan 2019 12:01:29 +0000 Subject: [PATCH 4/6] allow like for like constructor args --- lib/__tests__/cloudworker-e2e.test.js | 21 +++++++++++++++++++++ lib/runtime.js | 1 - lib/runtime/text-encoder.js | 8 ++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/__tests__/cloudworker-e2e.test.js b/lib/__tests__/cloudworker-e2e.test.js index 2098746..af6d008 100644 --- a/lib/__tests__/cloudworker-e2e.test.js +++ b/lib/__tests__/cloudworker-e2e.test.js @@ -248,4 +248,25 @@ describe('cloudworker-e2e', async () => { await server.close() cb() }) + + test('test ascii decoder can be specified', async (cb) => { + const script = ` + addEventListener('fetch', event => { + const euroSymbol = new Uint8Array([226, 130, 172]) + const decoder = new TextDecoder() + const asciiDecoder = new TextDecoder('ascii') + const sameDecodedValues = (decoder.decode(euroSymbol)) === (asciiDecoder.decode(euroSymbol)) + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + writer.write(new TextEncoder().encode(sameDecodedValues)).then(() => writer.close()) + event.respondWith(new Response(readable, {status: 200})) + }) + ` + const server = new Cloudworker(script).listen(8080) + const res = await axios.get('http://localhost:8080', defaultAxiosOpts) + expect(res.status).toEqual(200) + expect(res.data).toEqual(false) + await server.close() + cb() + }) }) diff --git a/lib/runtime.js b/lib/runtime.js index 9693aef..0807826 100644 --- a/lib/runtime.js +++ b/lib/runtime.js @@ -36,5 +36,4 @@ module.exports = { WritableStream, TransformStream, URL, - } diff --git a/lib/runtime/text-encoder.js b/lib/runtime/text-encoder.js index b241c3e..614ba0e 100644 --- a/lib/runtime/text-encoder.js +++ b/lib/runtime/text-encoder.js @@ -1,17 +1,17 @@ const textEncoding = require('text-encoding') -function CustomTextDecoder () { +function CustomTextDecoder (label, options) { if (!(this instanceof CustomTextDecoder)) { throw TypeError('Called as a function. Did you forget \'new\'?') } - this.decoder = new textEncoding.TextDecoder() + this.decoder = new textEncoding.TextDecoder(label, options) return this } CustomTextDecoder.prototype.decode = function decode (input, options) { if (Object.prototype.toString.call(input) === '[object Uint8Array]') { - var buffer = new ArrayBuffer(input.length) - var view = new Uint8Array(buffer) + const buffer = new ArrayBuffer(input.length) + const view = new Uint8Array(buffer) view.set(input) return this.decoder.decode( view, From 6e23cceb36cab9548a40f11c2cca95f41e90256d Mon Sep 17 00:00:00 2001 From: dominictootell Date: Tue, 15 Jan 2019 07:37:19 +0000 Subject: [PATCH 5/6] delegate encoding, fatal, and ignoreBOM --- lib/runtime/__tests__/textencoder.test.js | 28 +++++++++++++++++++++++ lib/runtime/text-encoder.js | 22 ++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 lib/runtime/__tests__/textencoder.test.js diff --git a/lib/runtime/__tests__/textencoder.test.js b/lib/runtime/__tests__/textencoder.test.js new file mode 100644 index 0000000..c89741b --- /dev/null +++ b/lib/runtime/__tests__/textencoder.test.js @@ -0,0 +1,28 @@ +const textEncoding = require('../text-encoder.js') + +describe('text-encoder', () => { + test('able to fetch encoding from text decoder', async () => { + const asciiDecoder = new textEncoding.TextDecoder('cp1251') + expect(await asciiDecoder.encoding).toEqual('windows-1251') + }) + + test('able to obtain ignoreBom from text decoder', async () => { + const asciiDecoder = new textEncoding.TextDecoder('ascii', { ignoreBOM: false }) + expect(await asciiDecoder.encoding).toEqual('windows-1252') + expect(await asciiDecoder.ignoreBOM).toEqual(false) + + const asciiDecoder2 = new textEncoding.TextDecoder('ascii', { ignoreBOM: true }) + expect(await asciiDecoder2.encoding).toEqual('windows-1252') + expect(await asciiDecoder2.ignoreBOM).toEqual(true) + }) + + test('able to obtain fatal from text decoder', async () => { + const asciiDecoder = new textEncoding.TextDecoder('ascii', { fatal: false }) + expect(await asciiDecoder.encoding).toEqual('windows-1252') + expect(await asciiDecoder.fatal).toEqual(false) + + const asciiDecoder2 = new textEncoding.TextDecoder('ascii', { fatal: true }) + expect(await asciiDecoder2.encoding).toEqual('windows-1252') + expect(await asciiDecoder2.fatal).toEqual(true) + }) +}) diff --git a/lib/runtime/text-encoder.js b/lib/runtime/text-encoder.js index 614ba0e..349565b 100644 --- a/lib/runtime/text-encoder.js +++ b/lib/runtime/text-encoder.js @@ -21,5 +21,27 @@ CustomTextDecoder.prototype.decode = function decode (input, options) { return this.decoder.decode(input, options) } +if (Object.defineProperty) { + // The encoding attribute's getter must return encoding's name. + Object.defineProperty(CustomTextDecoder.prototype, 'encoding', { + /** @this {TextDecoder} */ + get: function () { return this.decoder.encoding }, + }) + + // The fatal attribute's getter must return true if error mode + // is fatal, and false otherwise. + Object.defineProperty(CustomTextDecoder.prototype, 'fatal', { + /** @this {TextDecoder} */ + get: function () { return this.decoder.fatal }, + }) + + // The ignoreBOM attribute's getter must return true if ignore + // BOM flag is set, and false otherwise. + Object.defineProperty(CustomTextDecoder.prototype, 'ignoreBOM', { + /** @this {TextDecoder} */ + get: function () { return this.decoder.ignoreBOM }, + }) +} + module.exports.TextDecoder = CustomTextDecoder module.exports.TextEncoder = textEncoding.TextEncoder From 3a8d2baf90198274976d115dee2287017daa9b36 Mon Sep 17 00:00:00 2001 From: dominictootell Date: Tue, 15 Jan 2019 22:09:37 +0000 Subject: [PATCH 6/6] update to remove await, and comments --- lib/runtime/__tests__/textencoder.test.js | 24 +++++++++--------- lib/runtime/text-encoder.js | 31 +++++++++-------------- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/lib/runtime/__tests__/textencoder.test.js b/lib/runtime/__tests__/textencoder.test.js index c89741b..9e9f58a 100644 --- a/lib/runtime/__tests__/textencoder.test.js +++ b/lib/runtime/__tests__/textencoder.test.js @@ -1,28 +1,28 @@ const textEncoding = require('../text-encoder.js') describe('text-encoder', () => { - test('able to fetch encoding from text decoder', async () => { + test('able to fetch encoding from text decoder', () => { const asciiDecoder = new textEncoding.TextDecoder('cp1251') - expect(await asciiDecoder.encoding).toEqual('windows-1251') + expect(asciiDecoder.encoding).toEqual('windows-1251') }) - test('able to obtain ignoreBom from text decoder', async () => { + test('able to obtain ignoreBom from text decoder', () => { const asciiDecoder = new textEncoding.TextDecoder('ascii', { ignoreBOM: false }) - expect(await asciiDecoder.encoding).toEqual('windows-1252') - expect(await asciiDecoder.ignoreBOM).toEqual(false) + expect(asciiDecoder.encoding).toEqual('windows-1252') + expect(asciiDecoder.ignoreBOM).toEqual(false) const asciiDecoder2 = new textEncoding.TextDecoder('ascii', { ignoreBOM: true }) - expect(await asciiDecoder2.encoding).toEqual('windows-1252') - expect(await asciiDecoder2.ignoreBOM).toEqual(true) + expect(asciiDecoder2.encoding).toEqual('windows-1252') + expect(asciiDecoder2.ignoreBOM).toEqual(true) }) - test('able to obtain fatal from text decoder', async () => { + test('able to obtain fatal from text decoder', () => { const asciiDecoder = new textEncoding.TextDecoder('ascii', { fatal: false }) - expect(await asciiDecoder.encoding).toEqual('windows-1252') - expect(await asciiDecoder.fatal).toEqual(false) + expect(asciiDecoder.encoding).toEqual('windows-1252') + expect(asciiDecoder.fatal).toEqual(false) const asciiDecoder2 = new textEncoding.TextDecoder('ascii', { fatal: true }) - expect(await asciiDecoder2.encoding).toEqual('windows-1252') - expect(await asciiDecoder2.fatal).toEqual(true) + expect(asciiDecoder2.encoding).toEqual('windows-1252') + expect(asciiDecoder2.fatal).toEqual(true) }) }) diff --git a/lib/runtime/text-encoder.js b/lib/runtime/text-encoder.js index 349565b..5460ee9 100644 --- a/lib/runtime/text-encoder.js +++ b/lib/runtime/text-encoder.js @@ -21,27 +21,20 @@ CustomTextDecoder.prototype.decode = function decode (input, options) { return this.decoder.decode(input, options) } -if (Object.defineProperty) { - // The encoding attribute's getter must return encoding's name. - Object.defineProperty(CustomTextDecoder.prototype, 'encoding', { - /** @this {TextDecoder} */ - get: function () { return this.decoder.encoding }, - }) +Object.defineProperty(CustomTextDecoder.prototype, 'encoding', { + /** @this {CustomTextDecoder} */ + get: function () { return this.decoder.encoding }, +}) - // The fatal attribute's getter must return true if error mode - // is fatal, and false otherwise. - Object.defineProperty(CustomTextDecoder.prototype, 'fatal', { - /** @this {TextDecoder} */ - get: function () { return this.decoder.fatal }, - }) +Object.defineProperty(CustomTextDecoder.prototype, 'fatal', { + /** @this {CustomTextDecoder} */ + get: function () { return this.decoder.fatal }, +}) - // The ignoreBOM attribute's getter must return true if ignore - // BOM flag is set, and false otherwise. - Object.defineProperty(CustomTextDecoder.prototype, 'ignoreBOM', { - /** @this {TextDecoder} */ - get: function () { return this.decoder.ignoreBOM }, - }) -} +Object.defineProperty(CustomTextDecoder.prototype, 'ignoreBOM', { + /** @this {CustomTextDecoder} */ + get: function () { return this.decoder.ignoreBOM }, +}) module.exports.TextDecoder = CustomTextDecoder module.exports.TextEncoder = textEncoding.TextEncoder