diff --git a/acorn-loose/src/expression.js b/acorn-loose/src/expression.js index ca77f315d..4cfe1100d 100644 --- a/acorn-loose/src/expression.js +++ b/acorn-loose/src/expression.js @@ -266,7 +266,7 @@ lp.parseExprAtom = function() { node = this.startNode() node.value = this.tok.value node.raw = this.input.slice(this.tok.start, this.tok.end) - if (this.tok.type === tt.num && node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1) + if (this.tok.type === tt.num && node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1).replace(/_/g, "") this.next() return this.finishNode(node, "Literal") diff --git a/acorn/src/expression.js b/acorn/src/expression.js index 5b8c2093d..d71701374 100644 --- a/acorn/src/expression.js +++ b/acorn/src/expression.js @@ -517,7 +517,7 @@ pp.parseLiteral = function(value) { let node = this.startNode() node.value = value node.raw = this.input.slice(this.start, this.end) - if (node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1) + if (node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1).replace(/_/g, "") this.next() return this.finishNode(node, "Literal") } diff --git a/acorn/src/tokenize.js b/acorn/src/tokenize.js index 29e56b6be..6c86f5c51 100644 --- a/acorn/src/tokenize.js +++ b/acorn/src/tokenize.js @@ -438,30 +438,67 @@ pp.readRegexp = function() { // were read, the integer value otherwise. When `len` is given, this // will return `null` unless the integer has exactly `len` digits. -pp.readInt = function(radix, len) { - let start = this.pos, total = 0 - for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) { +pp.readInt = function(radix, len, maybeLegacyOctalNumericLiteral) { + // `len` is used for character escape sequences. In that case, disallow separators. + const allowSeparators = this.options.ecmaVersion >= 12 && len === undefined + + // `maybeLegacyOctalNumericLiteral` is true if it doesn't have prefix (0x,0o,0b) + // and isn't fraction part nor exponent part. In that case, if the first digit + // is zero then disallow separators. + const isLegacyOctalNumericLiteral = maybeLegacyOctalNumericLiteral && this.input.charCodeAt(this.pos) === 48 + + let start = this.pos, total = 0, lastCode = 0 + for (let i = 0, e = len == null ? Infinity : len; i < e; ++i, ++this.pos) { let code = this.input.charCodeAt(this.pos), val + + if (allowSeparators && code === 95) { + if (isLegacyOctalNumericLiteral) this.raiseRecoverable(this.pos, "Numeric separator is not allowed in legacy octal numeric literals") + if (lastCode === 95) this.raiseRecoverable(this.pos, "Numeric separator must be exactly one underscore") + if (i === 0) this.raiseRecoverable(this.pos, "Numeric separator is not allowed at the first of digits") + lastCode = code + continue + } + if (code >= 97) val = code - 97 + 10 // a else if (code >= 65) val = code - 65 + 10 // A else if (code >= 48 && code <= 57) val = code - 48 // 0-9 else val = Infinity if (val >= radix) break - ++this.pos + lastCode = code total = total * radix + val } + + if (allowSeparators && lastCode === 95) this.raiseRecoverable(this.pos - 1, "Numeric separator is not allowed at the last of digits") if (this.pos === start || len != null && this.pos - start !== len) return null return total } +function stringToNumber(str, isLegacyOctalNumericLiteral) { + if (isLegacyOctalNumericLiteral) { + return parseInt(str, 8) + } + + // `parseFloat(value)` stops parsing at the first numeric separator then returns a wrong value. + return parseFloat(str.replace(/_/g, "")) +} + +function stringToBigInt(str) { + if (typeof BigInt !== "function") { + return null + } + + // `BigInt(value)` throws syntax error if the string contains numeric separators. + return BigInt(str.replace(/_/g, "")) +} + pp.readRadixNumber = function(radix) { let start = this.pos this.pos += 2 // 0x let val = this.readInt(radix) if (val == null) this.raise(this.start + 2, "Expected number in radix " + radix) if (this.options.ecmaVersion >= 11 && this.input.charCodeAt(this.pos) === 110) { - val = typeof BigInt !== "undefined" ? BigInt(this.input.slice(start, this.pos)) : null + val = stringToBigInt(this.input.slice(start, this.pos)) ++this.pos } else if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number") return this.finishToken(tt.num, val) @@ -471,13 +508,12 @@ pp.readRadixNumber = function(radix) { pp.readNumber = function(startsWithDot) { let start = this.pos - if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number") + if (!startsWithDot && this.readInt(10, undefined, true) === null) this.raise(start, "Invalid number") let octal = this.pos - start >= 2 && this.input.charCodeAt(start) === 48 if (octal && this.strict) this.raise(start, "Invalid number") let next = this.input.charCodeAt(this.pos) if (!octal && !startsWithDot && this.options.ecmaVersion >= 11 && next === 110) { - let str = this.input.slice(start, this.pos) - let val = typeof BigInt !== "undefined" ? BigInt(str) : null + let val = stringToBigInt(this.input.slice(start, this.pos)) ++this.pos if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number") return this.finishToken(tt.num, val) @@ -495,8 +531,7 @@ pp.readNumber = function(startsWithDot) { } if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number") - let str = this.input.slice(start, this.pos) - let val = octal ? parseInt(str, 8) : parseFloat(str) + let val = stringToNumber(this.input.slice(start, this.pos), octal) return this.finishToken(tt.num, val) } diff --git a/bin/run_test262.js b/bin/run_test262.js index f88abb2d9..ff55bcd19 100644 --- a/bin/run_test262.js +++ b/bin/run_test262.js @@ -10,7 +10,6 @@ const unsupportedFeatures = [ "class-static-fields-private", "class-static-fields-public", "class-static-methods-private", - "numeric-separator-literal", "logical-assignment-operators", ]; diff --git a/test/run.js b/test/run.js index bd7cebd58..80d3ad42d 100644 --- a/test/run.js +++ b/test/run.js @@ -22,6 +22,7 @@ require("./tests-nullish-coalescing.js"); require("./tests-optional-chaining.js"); require("./tests-logical-assignment-operators.js"); + require("./tests-numeric-separators.js"); var acorn = require("../acorn") var acorn_loose = require("../acorn-loose") diff --git a/test/tests-numeric-separators.js b/test/tests-numeric-separators.js new file mode 100644 index 000000000..e28ef5055 --- /dev/null +++ b/test/tests-numeric-separators.js @@ -0,0 +1,199 @@ +// Tests for ECMAScript 2021 Numeric Separators + +if (typeof exports != 'undefined') { + var test = require('./driver.js').test; + var testFail = require('./driver.js').testFail; +} + +function bigint(str) { + if (typeof BigInt !== "function") { + return null + } + return BigInt(str) +} + +test( + "123_456", + { + "type": "Program", + "start": 0, + "end": 7, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 7, + "expression": { + "type": "Literal", + "start": 0, + "end": 7, + "value": 123456, + "raw": "123_456" + } + } + ], + "sourceType": "script" + }, + { ecmaVersion: 12 } +); + +test( + "123_456.123_456e+123_456", + { + "type": "Program", + "start": 0, + "end": 24, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 24, + "expression": { + "type": "Literal", + "start": 0, + "end": 24, + "value": 123456.123456e+123456, + "raw": "123_456.123_456e+123_456" + } + } + ], + "sourceType": "script" + }, + { ecmaVersion: 12 } +); + +test( + "0b1010_0001", + { + "type": "Program", + "start": 0, + "end": 11, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 11, + "expression": { + "type": "Literal", + "start": 0, + "end": 11, + "value": 0b10100001, + "raw": "0b1010_0001" + } + } + ], + "sourceType": "script" + }, + { ecmaVersion: 12 } +); + +test( + "0xDEAD_BEAF", + { + "type": "Program", + "start": 0, + "end": 11, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 11, + "expression": { + "type": "Literal", + "start": 0, + "end": 11, + "value": 0xDEADBEAF, + "raw": "0xDEAD_BEAF" + } + } + ], + "sourceType": "script" + }, + { ecmaVersion: 12 } +); + +test( + "0o755_666", + { + "type": "Program", + "start": 0, + "end": 9, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 9, + "expression": { + "type": "Literal", + "start": 0, + "end": 9, + "value": 0o755666, + "raw": "0o755_666" + } + } + ], + "sourceType": "script" + }, + { ecmaVersion: 12 } +); + +test( + "123_456n", + { + "type": "Program", + "start": 0, + "end": 8, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 8, + "expression": { + "type": "Literal", + "start": 0, + "end": 8, + "value": bigint("123456"), + "raw": "123_456n", + "bigint": "123456" + } + } + ], + "sourceType": "script" + }, + { ecmaVersion: 12 } +); + +test( + ".012_345", + { + "type": "Program", + "start": 0, + "end": 8, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 8, + "expression": { + "type": "Literal", + "start": 0, + "end": 8, + "value": 0.012345, + "raw": ".012_345" + } + } + ], + "sourceType": "script" + }, + { ecmaVersion: 12 } +); + +testFail("123_456", "Identifier directly after number (1:3)", { ecmaVersion: 11 }); +testFail("123__456", "Numeric separator must be exactly one underscore (1:4)", { ecmaVersion: 12 }); +testFail("0._123456", "Numeric separator is not allowed at the first of digits (1:2)", { ecmaVersion: 12 }); +testFail("123456_", "Numeric separator is not allowed at the last of digits (1:6)", { ecmaVersion: 12 }); +testFail("012_345", "Numeric separator is not allowed in legacy octal numeric literals (1:3)", { ecmaVersion: 12 }); + +testFail("'\\x2_0'", "Bad character escape sequence (1:3)", { ecmaVersion: 12 }); +testFail("'\\u00_20'", "Bad character escape sequence (1:3)", { ecmaVersion: 12 }); +testFail("'\\u{2_0}'", "Bad character escape sequence (1:4)", { ecmaVersion: 12 });