Skip to content
This repository has been archived by the owner on May 19, 2018. It is now read-only.

Fix: major revision to valid and invalid numeric literal separator "sibling" characters #745

Merged
merged 3 commits into from
Sep 29, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions src/tokenizer/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,48 @@ const forbiddenNumericSeparatorSiblings = {
],
};

const allowedNumericSeparatorSiblings = {};
allowedNumericSeparatorSiblings.bin = [
// 0 - 1
48,
49,
];
allowedNumericSeparatorSiblings.oct = [
// 0 - 7
...allowedNumericSeparatorSiblings.bin,
50,
51,
52,
53,
54,
55,
];
allowedNumericSeparatorSiblings.dec = [
// 0 - 9
...allowedNumericSeparatorSiblings.oct,
56,
57,
];

allowedNumericSeparatorSiblings.hex = [
// 0 - 9, A - F, a - f,
...allowedNumericSeparatorSiblings.dec,
// A - F
65,
66,
67,
68,
69,
70,
// a - f
97,
98,
99,
100,
101,
102,
];

// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// used for the onToken callback and the external tokenizer.
Expand Down Expand Up @@ -718,6 +760,15 @@ export default class Tokenizer extends LocationParser {
radix === 16
? forbiddenNumericSeparatorSiblings.hex
: forbiddenNumericSeparatorSiblings.decBinOct;
const allowedSiblings =
radix === 16
? allowedNumericSeparatorSiblings.hex
: radix === 10
? allowedNumericSeparatorSiblings.dec
: radix === 8
? allowedNumericSeparatorSiblings.oct
: allowedNumericSeparatorSiblings.bin;

let total = 0;

for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
Expand All @@ -728,6 +779,10 @@ export default class Tokenizer extends LocationParser {
const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.input.charCodeAt(this.state.pos + 1);
if (code === 95) {
if (allowedSiblings.indexOf(next) === -1) {
this.raise(this.state.pos, "Invalid or unexpected token");
}

if (
forbiddenSiblings.indexOf(prev) > -1 ||
forbiddenSiblings.indexOf(next) > -1 ||
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_
1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_1_
1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x1_1_
0x1_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1__)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1__1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1_.1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1._1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1_e1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1_E1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1e_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1E_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0xa_1_
0xa_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0xa_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_a_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x__1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_1__1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_11)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b_0_1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_a_1
0x_a_1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b_01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1__}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1__1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1_.1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1._1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x__1_1_
0x__1_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1_e1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1_E1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1e_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1E_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0xa_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_a_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x__1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_1__1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_1__1
0x_1__1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_11}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b_0_1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b_01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_1_1_
0x_1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_1_1
0o_1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_11
0o_11
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_01_1_
0o_01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b_0_1_1
0b_0_1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_1__
1_1__
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b_01_1_
0b_01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b01_1_
0b01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o1_1_
0o1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_1_1_
0o_1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
._1_1
._1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Unexpected token (1:0)" }
{ "throws": "Unexpected token (1:0)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o01_8
0o01_8
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Unexpected token, expected ; (1:5)" }
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b2_1
0b2_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Expected number in radix 2 (1:2)" }
{ "throws": "Expected number in radix 2 (1:2)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0xZ_1
0xZ_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Expected number in radix 16 (1:2)" }
{ "throws": "Expected number in radix 16 (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:1)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1__1
1__1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1__;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1__1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:1)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_.1_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1._1_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1_e1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Loading