Skip to content

Commit

Permalink
test: split encoding tests where some cases can be run without ICU
Browse files Browse the repository at this point in the history
Split the following tests:

- `test-whatwg-encoding-textdecoder-utf16-surrogates.js`
- `test-whatwg-encoding-textdecoder-ignorebom.js`
- `test-whatwg-encoding-textdecoder-streaming.js`

Each into two files: one that can be run without ICU and one that has
to be run with ICU. The latter can be replaced with WPT later.

PR-URL: #25155
Reviewed-By: James M Snell <[email protected]>
  • Loading branch information
joyeecheung authored and targos committed Jan 1, 2019
1 parent deceb26 commit be42182
Show file tree
Hide file tree
Showing 6 changed files with 82 additions and 11 deletions.
30 changes: 30 additions & 0 deletions test/parallel/test-whatwg-encoding-custom-textdecoder-ignorebom.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
'use strict';

// From: https://github.com/w3c/web-platform-tests/blob/7f567fa29c/encoding/textdecoder-ignorebom.html
// This is the part that can be run without ICU

require('../common');

const assert = require('assert');

const cases = [
{
encoding: 'utf-8',
bytes: [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63]
},
{
encoding: 'utf-16le',
bytes: [0xFF, 0xFE, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00]
}
];

cases.forEach((testCase) => {
const BOM = '\uFEFF';
let decoder = new TextDecoder(testCase.encoding, { ignoreBOM: true });
const bytes = new Uint8Array(testCase.bytes);
assert.strictEqual(decoder.decode(bytes), `${BOM}abc`);
decoder = new TextDecoder(testCase.encoding, { ignoreBOM: false });
assert.strictEqual(decoder.decode(bytes), 'abc');
decoder = new TextDecoder(testCase.encoding);
assert.strictEqual(decoder.decode(bytes), 'abc');
});
38 changes: 38 additions & 0 deletions test/parallel/test-whatwg-encoding-custom-textdecoder-streaming.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
'use strict';

// From: https://github.com/w3c/web-platform-tests/blob/fa9436d12c/encoding/textdecoder-streaming.html
// This is the part that can be run without ICU

require('../common');

const assert = require('assert');

const string =
'\x00123ABCabc\x80\xFF\u0100\u1000\uFFFD\uD800\uDC00\uDBFF\uDFFF';
const octets = {
'utf-8': [
0x00, 0x31, 0x32, 0x33, 0x41, 0x42, 0x43, 0x61, 0x62, 0x63, 0xc2, 0x80,
0xc3, 0xbf, 0xc4, 0x80, 0xe1, 0x80, 0x80, 0xef, 0xbf, 0xbd, 0xf0, 0x90,
0x80, 0x80, 0xf4, 0x8f, 0xbf, 0xbf],
'utf-16le': [
0x00, 0x00, 0x31, 0x00, 0x32, 0x00, 0x33, 0x00, 0x41, 0x00, 0x42, 0x00,
0x43, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00, 0x80, 0x00, 0xFF, 0x00,
0x00, 0x01, 0x00, 0x10, 0xFD, 0xFF, 0x00, 0xD8, 0x00, 0xDC, 0xFF, 0xDB,
0xFF, 0xDF]
};

Object.keys(octets).forEach((encoding) => {
for (let len = 1; len <= 5; ++len) {
const encoded = octets[encoding];
const decoder = new TextDecoder(encoding);
let out = '';
for (let i = 0; i < encoded.length; i += len) {
const sub = [];
for (let j = i; j < encoded.length && j < i + len; ++j)
sub.push(encoded[j]);
out += decoder.decode(new Uint8Array(sub), { stream: true });
}
out += decoder.decode();
assert.strictEqual(out, string);
}
});
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use strict';

// From: https://github.com/w3c/web-platform-tests/blob/39a67e2fff/encoding/textdecoder-utf16-surrogates.html
// With the twist that we specifically test for Node.js error codes

const common = require('../common');

Expand Down Expand Up @@ -43,7 +44,7 @@ const bad = [
];

bad.forEach((t) => {

// TODO(joyeecheung): remove this when WPT is ported
assert.strictEqual(
new TextDecoder(t.encoding).decode(new Uint8Array(t.input)),
t.expected);
Expand Down
17 changes: 7 additions & 10 deletions test/parallel/test-whatwg-encoding-textdecoder-ignorebom.js
Original file line number Diff line number Diff line change
@@ -1,34 +1,31 @@
'use strict';

// From: https://github.com/w3c/web-platform-tests/blob/7f567fa29c/encoding/textdecoder-ignorebom.html
// TODO(joyeecheung): replace this with WPT

const common = require('../common');

if (!common.hasIntl)
common.skip('missing Intl');

const assert = require('assert');

const cases = [
{
encoding: 'utf-8',
bytes: [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63],
skipNoIntl: false
bytes: [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63]
},
{
encoding: 'utf-16le',
bytes: [0xFF, 0xFE, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00],
skipNoIntl: false
bytes: [0xFF, 0xFE, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00]
},
{
encoding: 'utf-16be',
bytes: [0xFE, 0xFF, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63],
skipNoIntl: true
bytes: [0xFE, 0xFF, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63]
}
];

cases.forEach((testCase) => {
if (testCase.skipNoIntl && !common.hasIntl) {
console.log(`skipping ${testCase.encoding} because missing Intl`);
return; // skipping
}
const BOM = '\uFEFF';
let decoder = new TextDecoder(testCase.encoding, { ignoreBOM: true });
const bytes = new Uint8Array(testCase.bytes);
Expand Down
4 changes: 4 additions & 0 deletions test/parallel/test-whatwg-encoding-textdecoder-streaming.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
'use strict';

// From: https://github.com/w3c/web-platform-tests/blob/fa9436d12c/encoding/textdecoder-streaming.html
// TODO(joyeecheung): replace this with WPT

const common = require('../common');

if (!common.hasIntl)
common.skip('missing Intl');

const assert = require('assert');

const string =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use strict';

// From: https://github.com/w3c/web-platform-tests/blob/fa9436d12c/encoding/textencoder-utf16-surrogates.html
// TODO(joyeecheung): replace this with WPT

require('../common');

Expand Down

0 comments on commit be42182

Please sign in to comment.