From fd0f689828f17ba44237e660c548522a510371d8 Mon Sep 17 00:00:00 2001 From: Alan Pierce Date: Sun, 31 Mar 2019 10:17:16 -0700 Subject: [PATCH] Port all babel-parser changes from 2019-01-28 to 2019-03-26 (#440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #437 Details: 828169e61 Fix line continuation with Unicode line terminators (#9403) 🚫 We don't do escape parsing anyway. 4c4c22a31 Run prettier 🚫 Tools only. 00c3e3c8e Fixed link to @babel/parser's issues in README (#9427) 🚫 Docs only. 9eb010da5 Unify reserved word checking and update error messages (#9402) 🚫 Error handling only. 344d35bbe Simplify await and yield tracking in params (#9405) 🚫 Error handling only. d896ce2b5 v7.3.2 🚫 Release only. e03e5ba01 Add TypeScript definitions for parser plugin options. (#9457) 🚫 Types only. 07b0f22a3 Fix range for TypeScript optional parameter in arrow function (#9463) 🚫 AST only. d1514f57b Typescript function destructured params (#9431) 🚫 Already fixed in my code (with a fix that I think is better). 2817844e8 Fix regression with let (#9477) 🚫 This doesn't seem to affect Sucrase. d349b74a4 Better error output in parser tests (#9491) 🚫 Test only. 4ba998c5d Add importKind to spec 🚫 Types only. d1fe2d05f v7.3.3 🚫 Release only. 058f05742 Also check AssignmentPatterns for export name (#9521) 🚫 Error handling only. a1ea765b9 Make tests spec compliant and avoid duplicate declarations in input files (#9522) 🚫 Test only. dd8b700a2 Parenthesized expressions (#8025) 🚫 New feature that won't go into Sucrase. 9f3457797 Fix TypeScript parsers missing token check (#9571) (#9572) ✅ Added new check with test. fc1ea7f49 Revert "Parenthesized expressions (#8025)" 🚫 New feature that won't go into Sucrase. 1f6454cc9 v7.3.4 🚫 Release only. a7391144b Introduce scope tracking in the parser (#9493) 🚫 Looks like this is just for error handling. e6c1065d1 Fix strict mode prescanning with EmptyStatement (#9585) 🚫 We don't detect strict mode. d0e196d21 Treat for loop body as part of loop scope (#9586) 🚫 Scopes not tracked in Sucrase in the same way. 0345c1bc1 Use `for..of Object.keys` instead of `for..in` (#9518) 🚫 Not relevant for Sucrase. 244e4580e Remove always false param allowExpressionBody (#9591) ✅ Removed, plus removed some other unnecessary params. a029071b8 [TS] Correctly forget `await`s after parsing async arrows with type args (#9593) 🚫 Doesn't come up in Sucrase. e883ff295 Merge pull request #9597 from danez/Update-charcodes 🚫 Mostly just changes in types. 43eed1ac9 Check exported bindings are defined (#9589) 🚫 Error handling only. 5cb280f98 Fix scope check for 2nd+ lexical bindings (#9600) 🚫 Error handling only. 208195f42 Disallow duplicate params in methods (#9599) 🚫 Error handling only. 98ab1b642 Refactor parsing object members (#9607) 🚫 I won't try to port this refactor for now. f13f4adcb [TS] Disallow type casts in arrow parameters (#9612) 🚫 Error handling only. 17f4195bc Allow any reserved word in `export {} from` specifiers (#9616) 🚫 Already works in Sucrase. c60c4dd37 Partial Application Syntax: Stage 1 (#9343) ✅ Added basic parsing for ? expression. d832c0f43 Add parser support for placeholders (#9364) 🚫 I won't include this feature in Sucrase. 54ba6d80c Update identifier parsing per Unicode v12 (#9637) 🚫 Sucrase doesn't need this validation. 29999007f Disallow escape sequences in contextual keywords (#9618) 🚫 We don't support identifier escape sequences in the first place. fba5655a4 Parenthesized expressions (#8025) 🚫 I won't include this feature in Sucrase. e53be4b38 [TS] Allow context type annotation on getters/setters (#9641) 🚫 Bug in validation that Sucrase doesn't do. d8a532983 Reorganize token types and use a map for them (#9645) 🚫 Not really relevant for current Sucrase code. cf4bd8bb8 Remove input and length from state (#9646) 🚫 Not relevant for Sucrase. 29cd27b54 Partial application plugin (#9474) 🚫 Tests only. 25a3825a1 TypeScript Constant contexts (#9534) ✅ Added test, already works from previous change to make it an identifier. cc4560842 Add `readonly` to TypeScript type modifier (#9529) ✅ Added new case with test. 48d66eb64 Correctly parse TS TypeAssertions around arrow functions (#9699) 🚫 AST only, I think. f1328fb91 v7.4.0 🚫 Release only. ab41cb2cd Fix scope checks with enabled flow plugin (#9719) 🚫 Scope code doesn't exist in Sucrase. 2201fd839 Modules might be in loose mode when checking for undecl exports (#9725) 🚫 Sucrase always uses strict mode. 7dea0f23d v7.4.2 🚫 Release only. ef0722b4b Fix compatibility between estree and TS plugin (#9700) 🚫 Sucrase doesn't support estree mode. aaefc83a6 Allow HTML comments on first line (#9760) 🚫 Unclear what this is needed for, but doesn't seem important in Sucrase. d720c6cff Explicit labels for tokenTypes (#9761) 🚫 Internal change. 444daf922 Optimize parseBindingAtom code to get better error messages (#9762) 🚫 Error checking only. 2867bbf19 [typescript] parsing template literal as type (#9748) ✅ Added new case with test. 7f4427432 Parse right-hand-side of for/of as an assignment expression (#9767) 🚫 Error checking only. 6bc9e7ebd Correctly check for-in and for-of loop for invalid left-hand side (#9768) 🚫 Error checking only. 60d7e940e Fix merge error 🚫 Not relevant to Sucrase. --- src/parser/plugins/flow.ts | 17 +++----- src/parser/plugins/typescript.ts | 30 ++++++------- src/parser/traverser/expression.ts | 68 ++++++++++-------------------- src/parser/traverser/statement.ts | 29 +++++-------- test/sucrase-test.ts | 12 ++++++ test/tokens-test.ts | 30 ++++++++++++- test/typescript-test.ts | 35 +++++++++++++++ 7 files changed, 130 insertions(+), 91 deletions(-) diff --git a/src/parser/plugins/flow.ts b/src/parser/plugins/flow.ts index dfa10ab9..6996ae4f 100644 --- a/src/parser/plugins/flow.ts +++ b/src/parser/plugins/flow.ts @@ -709,18 +709,13 @@ export function flowParseVariance(): void { // Overrides // ================================== -export function flowParseFunctionBodyAndFinish( - functionStart: number, - isGenerator: boolean, - allowExpressionBody: boolean = false, - funcContextId: number, -): void { +export function flowParseFunctionBodyAndFinish(funcContextId: number): void { // For arrow functions, `parseArrow` handles the return type itself. - if (!allowExpressionBody && match(tt.colon)) { + if (match(tt.colon)) { flowParseTypeAndPredicateInitialiser(); } - parseFunctionBody(functionStart, isGenerator, allowExpressionBody, funcContextId); + parseFunctionBody(false, funcContextId); } export function flowParseSubscript(startPos: number, noCalls: boolean, stopState: StopState): void { @@ -1029,7 +1024,7 @@ export function flowParseSubscripts(startPos: number, noCalls: boolean = false): match(tt.lessThan) ) { const snapshot = state.snapshot(); - const wasArrow = parseAsyncArrowWithTypeParameters(startPos); + const wasArrow = parseAsyncArrowWithTypeParameters(); if (wasArrow && !state.error) { return; } @@ -1040,13 +1035,13 @@ export function flowParseSubscripts(startPos: number, noCalls: boolean = false): } // Returns true if there was an arrow function here. -function parseAsyncArrowWithTypeParameters(startPos: number): boolean { +function parseAsyncArrowWithTypeParameters(): boolean { state.scopeDepth++; const startTokenIndex = state.tokens.length; parseFunctionParams(); if (!parseArrow()) { return false; } - parseArrowExpression(startPos, startTokenIndex); + parseArrowExpression(startTokenIndex); return true; } diff --git a/src/parser/plugins/typescript.ts b/src/parser/plugins/typescript.ts index 7c580267..6414de43 100644 --- a/src/parser/plugins/typescript.ts +++ b/src/parser/plugins/typescript.ts @@ -73,7 +73,8 @@ function tsNextTokenCanFollowModifier(): boolean { !match(tt.parenR) && !match(tt.colon) && !match(tt.eq) && - !match(tt.question); + !match(tt.question) && + !match(tt.bang); if (canFollowModifier) { return true; @@ -448,6 +449,9 @@ function tsParseNonArrayType(): void { case tt.parenL: tsParseParenthesizedType(); return; + case tt.backQuote: + parseTemplate(); + return; default: if (state.type & TokenType.IS_KEYWORD) { next(); @@ -477,7 +481,11 @@ function tsParseInferType(): void { } function tsParseTypeOperatorOrHigher(): void { - if (isContextual(ContextualKeyword._keyof) || isContextual(ContextualKeyword._unique)) { + if ( + isContextual(ContextualKeyword._keyof) || + isContextual(ContextualKeyword._unique) || + isContextual(ContextualKeyword._readonly) + ) { next(); tsParseTypeOperatorOrHigher(); } else if (isContextual(ContextualKeyword._infer)) { @@ -969,10 +977,7 @@ function tsTryParseGenericAsyncArrowFunction(): boolean { return false; } - // We don't need to be precise about the function start since it's only used if this is a - // bodiless function, which isn't valid here. - const functionStart = state.start; - parseFunctionBody(functionStart, false /* isGenerator */, true); + parseFunctionBody(true); return true; } @@ -1009,21 +1014,16 @@ export function tsIsDeclarationStart(): boolean { // OVERRIDES // ====================================================== -export function tsParseFunctionBodyAndFinish( - functionStart: number, - isGenerator: boolean, - allowExpressionBody: boolean = false, - funcContextId: number, -): void { +export function tsParseFunctionBodyAndFinish(functionStart: number, funcContextId: number): void { // For arrow functions, `parseArrow` handles the return type itself. - if (!allowExpressionBody && match(tt.colon)) { + if (match(tt.colon)) { tsParseTypeOrTypePredicateAnnotation(tt.colon); } // The original code checked the node type to make sure this function type allows a missing // body, but we skip that to avoid sending around the node type. We instead just use the // allowExpressionBody boolean to make sure it's not an arrow function. - if (!allowExpressionBody && !match(tt.braceL) && isLineTerminator()) { + if (!match(tt.braceL) && isLineTerminator()) { // Retroactively mark the function declaration as a type. let i = state.tokens.length - 1; while ( @@ -1038,7 +1038,7 @@ export function tsParseFunctionBodyAndFinish( return; } - parseFunctionBody(functionStart, isGenerator, allowExpressionBody, funcContextId); + parseFunctionBody(false, funcContextId); } export function tsParseSubscript(startPos: number, noCalls: boolean, stopState: StopState): void { diff --git a/src/parser/traverser/expression.ts b/src/parser/traverser/expression.ts index e0f4b56c..7d248628 100644 --- a/src/parser/traverser/expression.ts +++ b/src/parser/traverser/expression.ts @@ -382,7 +382,7 @@ function parseAsyncArrowFromCallExpression(functionStart: number, startTokenInde flowStartParseAsyncArrowFromCallExpression(); } expect(tt.arrow); - parseArrowExpression(functionStart, startTokenIndex); + parseArrowExpression(startTokenIndex); } // Parse a no-call expression (like argument of `new` or `::` operators). @@ -450,7 +450,7 @@ export function parseExprAtom(): boolean { !canInsertSemicolon() ) { next(); - parseFunction(functionStart, false, false); + parseFunction(functionStart, false); return false; } else if ( canBeArrow && @@ -462,7 +462,7 @@ export function parseExprAtom(): boolean { parseBindingIdentifier(false); expect(tt.arrow); // let foo = async bar => {}; - parseArrowExpression(functionStart, startTokenIndex); + parseArrowExpression(startTokenIndex); return true; } @@ -470,7 +470,7 @@ export function parseExprAtom(): boolean { state.scopeDepth++; markPriorBindingIdentifier(false); expect(tt.arrow); - parseArrowExpression(functionStart, startTokenIndex); + parseArrowExpression(startTokenIndex); return true; } @@ -611,12 +611,10 @@ function parseParenAndDistinguishExpression(canBeArrow: boolean): boolean { // get proper token annotations. state.restoreFromSnapshot(snapshot); state.scopeDepth++; - // We don't need to worry about functionStart for arrow functions, so just use something. - const functionStart = state.start; // Don't specify a context ID because arrow functions don't need a context ID. parseFunctionParams(); parseArrow(); - parseArrowExpression(functionStart, startTokenIndex); + parseArrowExpression(startTokenIndex); return true; } } @@ -751,7 +749,7 @@ export function parseObj(isPattern: boolean, isBlockScope: boolean): void { parsePropertyName(contextId); } - parseObjPropValue(isGenerator, isPattern, isBlockScope, contextId); + parseObjPropValue(isPattern, isBlockScope, contextId); } state.tokens[state.tokens.length - 1].contextId = contextId; @@ -771,23 +769,19 @@ function isGetterOrSetterMethod(isPattern: boolean): boolean { } // Returns true if this was a method. -function parseObjectMethod( - isGenerator: boolean, - isPattern: boolean, - objectContextId: number, -): boolean { +function parseObjectMethod(isPattern: boolean, objectContextId: number): boolean { // We don't need to worry about modifiers because object methods can't have optional bodies, so // the start will never be used. const functionStart = state.start; if (match(tt.parenL)) { if (isPattern) unexpected(); - parseMethod(functionStart, isGenerator, /* isConstructor */ false); + parseMethod(functionStart, /* isConstructor */ false); return true; } if (isGetterOrSetterMethod(isPattern)) { parsePropertyName(objectContextId); - parseMethod(functionStart, /* isGenerator */ false, /* isConstructor */ false); + parseMethod(functionStart, /* isConstructor */ false); return true; } return false; @@ -822,7 +816,6 @@ function parseObjectProperty(isPattern: boolean, isBlockScope: boolean): void { } function parseObjPropValue( - isGenerator: boolean, isPattern: boolean, isBlockScope: boolean, objectContextId: number, @@ -832,7 +825,7 @@ function parseObjPropValue( } else if (isFlowEnabled) { flowStartParseObjPropValue(); } - const wasMethod = parseObjectMethod(isGenerator, isPattern, objectContextId); + const wasMethod = parseObjectMethod(isPattern, objectContextId); if (!wasMethod) { parseObjectProperty(isPattern, isBlockScope); } @@ -860,23 +853,14 @@ export function parsePropertyName(objectContextId: number): void { } // Parse object or class method. -export function parseMethod( - functionStart: number, - isGenerator: boolean, - isConstructor: boolean, -): void { +export function parseMethod(functionStart: number, isConstructor: boolean): void { const funcContextId = getNextContextId(); state.scopeDepth++; const startTokenIndex = state.tokens.length; const allowModifiers = isConstructor; // For TypeScript parameter properties parseFunctionParams(allowModifiers, funcContextId); - parseFunctionBodyAndFinish( - functionStart, - isGenerator, - false /* allowExpressionBody */, - funcContextId, - ); + parseFunctionBodyAndFinish(functionStart, funcContextId); const endTokenIndex = state.tokens.length; state.scopes.push(new Scope(startTokenIndex, endTokenIndex, true)); state.scopeDepth--; @@ -885,35 +869,24 @@ export function parseMethod( // Parse arrow function expression. // If the parameters are provided, they will be converted to an // assignable list. -export function parseArrowExpression(functionStart: number, startTokenIndex: number): void { - parseFunctionBody(functionStart, false /* isGenerator */, true); +export function parseArrowExpression(startTokenIndex: number): void { + parseFunctionBody(true); const endTokenIndex = state.tokens.length; state.scopes.push(new Scope(startTokenIndex, endTokenIndex, true)); state.scopeDepth--; } -export function parseFunctionBodyAndFinish( - functionStart: number, - isGenerator: boolean, - allowExpressionBody: boolean = false, - funcContextId: number = 0, -): void { +export function parseFunctionBodyAndFinish(functionStart: number, funcContextId: number = 0): void { if (isTypeScriptEnabled) { - tsParseFunctionBodyAndFinish(functionStart, isGenerator, allowExpressionBody, funcContextId); + tsParseFunctionBodyAndFinish(functionStart, funcContextId); } else if (isFlowEnabled) { - flowParseFunctionBodyAndFinish(functionStart, isGenerator, allowExpressionBody, funcContextId); + flowParseFunctionBodyAndFinish(funcContextId); } else { - parseFunctionBody(functionStart, isGenerator, allowExpressionBody, funcContextId); + parseFunctionBody(false, funcContextId); } } -// Parse function body and check parameters. -export function parseFunctionBody( - functionStart: number, - isGenerator: boolean, - allowExpression: boolean, - funcContextId: number = 0, -): void { +export function parseFunctionBody(allowExpression: boolean, funcContextId: number = 0): void { const isExpression = allowExpression && !match(tt.braceL); if (isExpression) { @@ -948,6 +921,9 @@ function parseExprListItem(allowEmpty: boolean): void { } else if (match(tt.ellipsis)) { parseSpread(); parseParenItem(); + } else if (match(tt.question)) { + // Partial function application proposal. + next(); } else { parseMaybeAssign(false, true); } diff --git a/src/parser/traverser/statement.ts b/src/parser/traverser/statement.ts index 6bbce039..7055a05b 100644 --- a/src/parser/traverser/statement.ts +++ b/src/parser/traverser/statement.ts @@ -198,7 +198,7 @@ function parseStatementContent(declaration: boolean): void { next(); if (match(tt._function) && !canInsertSemicolon()) { expect(tt._function); - parseFunction(functionStart, true, false); + parseFunction(functionStart, true); return; } else { state.restoreFromSnapshot(snapshot); @@ -563,12 +563,9 @@ function parseVarHead(isBlockScope: boolean): void { export function parseFunction( functionStart: number, isStatement: boolean, - allowExpressionBody: boolean = false, optionalId: boolean = false, ): void { - let isGenerator = false; if (match(tt.star)) { - isGenerator = true; next(); } @@ -591,7 +588,7 @@ export function parseFunction( const startTokenIndex = state.tokens.length; state.scopeDepth++; parseFunctionParams(); - parseFunctionBodyAndFinish(functionStart, isGenerator, allowExpressionBody); + parseFunctionBodyAndFinish(functionStart); const endTokenIndex = state.tokens.length; // In addition to the block scope of the function body, we need a separate function-style scope // that includes the params. @@ -691,7 +688,7 @@ function parseClassMember(memberStart: number, classContextId: number): void { if (match(tt.name) && state.contextualKeyword === ContextualKeyword._static) { parseIdentifier(); // eats 'static' if (isClassMethod()) { - parseClassMethod(memberStart, false, /* isConstructor */ false); + parseClassMethod(memberStart, /* isConstructor */ false); return; } else if (isClassProperty()) { parseClassProperty(); @@ -718,7 +715,7 @@ function parseClassMemberWithIsStatic( if (eat(tt.star)) { // a generator parseClassPropertyName(classContextId); - parseClassMethod(memberStart, true, /* isConstructor */ false); + parseClassMethod(memberStart, /* isConstructor */ false); return; } @@ -734,7 +731,7 @@ function parseClassMemberWithIsStatic( parsePostMemberNameModifiers(); if (isClassMethod()) { - parseClassMethod(memberStart, false, isConstructor); + parseClassMethod(memberStart, isConstructor); } else if (isClassProperty()) { parseClassProperty(); } else if (token.contextualKeyword === ContextualKeyword._async && !isLineTerminator()) { @@ -747,7 +744,7 @@ function parseClassMemberWithIsStatic( // The so-called parsed name would have been "async": get the real name. parseClassPropertyName(classContextId); - parseClassMethod(memberStart, isGenerator, false /* isConstructor */); + parseClassMethod(memberStart, false /* isConstructor */); } else if ( (token.contextualKeyword === ContextualKeyword._get || token.contextualKeyword === ContextualKeyword._set) && @@ -762,7 +759,7 @@ function parseClassMemberWithIsStatic( // a getter or setter // The so-called parsed name would have been "get/set": get the real name. parseClassPropertyName(classContextId); - parseClassMethod(memberStart, false, /* isConstructor */ false); + parseClassMethod(memberStart, /* isConstructor */ false); } else if (isLineTerminator()) { // an uninitialized class property (due to ASI, since we don't otherwise recognize the next token) parseClassProperty(); @@ -771,11 +768,7 @@ function parseClassMemberWithIsStatic( } } -function parseClassMethod( - functionStart: number, - isGenerator: boolean, - isConstructor: boolean, -): void { +function parseClassMethod(functionStart: number, isConstructor: boolean): void { if (isTypeScriptEnabled) { tsTryParseTypeParameters(); } else if (isFlowEnabled) { @@ -783,7 +776,7 @@ function parseClassMethod( flowParseTypeParameterDeclaration(); } } - parseMethod(functionStart, isGenerator, isConstructor); + parseMethod(functionStart, isConstructor); } // Return the name of the class property, if it is a simple identifier. @@ -901,12 +894,12 @@ function parseExportDefaultExpression(): void { } const functionStart = state.start; if (eat(tt._function)) { - parseFunction(functionStart, true, false, true); + parseFunction(functionStart, true, true); } else if (isContextual(ContextualKeyword._async) && lookaheadType() === tt._function) { // async function declaration eatContextual(ContextualKeyword._async); eat(tt._function); - parseFunction(functionStart, true, false, true); + parseFunction(functionStart, true, true); } else if (match(tt._class)) { parseClass(true, true); } else if (match(tt.at)) { diff --git a/test/sucrase-test.ts b/test/sucrase-test.ts index 582efdf6..03554a0b 100644 --- a/test/sucrase-test.ts +++ b/test/sucrase-test.ts @@ -797,4 +797,16 @@ describe("sucrase", () => { {transforms: []}, ); }); + + it("handles partial application syntax", () => { + assertResult( + ` + foo(?) + `, + ` + foo(?) + `, + {transforms: []}, + ); + }); }); diff --git a/test/tokens-test.ts b/test/tokens-test.ts index 1886d8a4..bc4dd80c 100644 --- a/test/tokens-test.ts +++ b/test/tokens-test.ts @@ -7,7 +7,7 @@ type SimpleToken = Token & {label?: string}; type TokenExpectation = {[K in keyof SimpleToken]?: SimpleToken[K]}; function assertTokens(code: string, expectedTokens: Array): void { - const tokens: Array = parse(code, true, false, false).tokens; + const tokens: Array = parse(code, true, true, false).tokens; assert.strictEqual(tokens.length, expectedTokens.length); const projectedTokens = tokens.map((token, i) => { const result = {}; @@ -275,4 +275,32 @@ describe("tokens", () => { ], ); }); + + it("properly parses keyword keys in TS class bodies", () => { + assertTokens( + ` + class A { + abstract?: void; + readonly!: void; + } + `, + [ + {type: tt._class}, + {type: tt.name}, + {type: tt.braceL}, + {type: tt.name}, + {type: tt.question}, + {type: tt.colon}, + {type: tt._void}, + {type: tt.semi}, + {type: tt.name}, + {type: tt.bang}, + {type: tt.colon}, + {type: tt._void}, + {type: tt.semi}, + {type: tt.braceR}, + {type: tt.eof}, + ], + ); + }); }); diff --git a/test/typescript-test.ts b/test/typescript-test.ts index 6997a4ec..60f2d906 100644 --- a/test/typescript-test.ts +++ b/test/typescript-test.ts @@ -1710,4 +1710,39 @@ describe("typescript transform", () => { `, ); }); + + it("handles const contexts", () => { + assertTypeScriptResult( + ` + let x = 5 as const; + `, + `"use strict"; + let x = 5 ; + `, + ); + }); + + it("handles the readonly type modifier", () => { + assertTypeScriptResult( + ` + let z: readonly number[]; + let z1: readonly [number, number]; + `, + `"use strict"; + let z; + let z1; + `, + ); + }); + + it("allows template literal syntax for type literals", () => { + assertTypeScriptResult( + ` + let x: \`foo\`; + `, + `"use strict"; + let x; + `, + ); + }); });