Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

perf(es/lexer): Optimize lexer #9075

Merged
merged 4 commits into from
Jun 18, 2024
+46 −53
Merged

perf(es/lexer): Optimize lexer #9075

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1fb55a2
state
kdy1 Jun 18, 2024
5825f96
repr
kdy1 Jun 18, 2024
716b83e
&dyn
kdy1 Jun 18, 2024
5e86120
Merge branch 'main' into pp
kdy1 Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/swc_ecma_parser/src/lexer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,7 +754,7 @@ impl<'a> Lexer<'a> {
/// character.
fn read_word_with(
&mut self,
convert: impl FnOnce(&str) -> Option<Word>,
convert: &dyn Fn(&str) -> Option<Word>,
) -> LexResult<Option<Token>> {
debug_assert!(self.cur().is_some());

Expand Down
55 changes: 24 additions & 31 deletions crates/swc_ecma_parser/src/lexer/state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ pub(super) struct State {
pub cur_line: usize,
pub line_start: BytePos,
pub prev_hi: BytePos,
pub tpl_start: BytePos,

context: TokenContexts,
syntax: Syntax,
Expand Down Expand Up @@ -323,11 +324,9 @@ impl<'a> Iterator for Lexer<'a> {
}
}

if let Some(TokenContext::Tpl {
start: start_pos_of_tpl,
}) = self.state.context.current()
{
return self.read_tmpl_token(start_pos_of_tpl).map(Some);
if let Some(TokenContext::Tpl {}) = self.state.context.current() {
let start = self.state.tpl_start;
return self.read_tmpl_token(start).map(Some);
}

self.read_token()
Expand Down Expand Up @@ -373,16 +372,17 @@ impl State {
State {
is_expr_allowed: true,
next_regexp: None,
is_first: true,
had_line_break: false,
had_line_break_before_last: false,
prev_hi: start_pos,
context,
token_type: None,
is_first: true,
start: BytePos(0),
line_start: BytePos(0),
cur_line: 1,
line_start: BytePos(0),
prev_hi: start_pos,
tpl_start: BytePos::DUMMY,
context,
syntax,
token_type: None,
}
}
}
Expand Down Expand Up @@ -431,30 +431,26 @@ impl State {
let prev = self.token_type.take();
self.token_type = Some(TokenType::from(next));

self.is_expr_allowed = Self::is_expr_allowed_on_next(
&mut self.context,
self.syntax,
prev,
start,
next,
self.had_line_break,
self.had_line_break_before_last,
self.is_expr_allowed,
);
self.is_expr_allowed = self.is_expr_allowed_on_next(prev, start, next);
}

/// `is_expr_allowed`: previous value.
/// `start`: start of newly produced token.
fn is_expr_allowed_on_next(
context: &mut TokenContexts,
syntax: Syntax,
&mut self,
prev: Option<TokenType>,
start: BytePos,
next: TokenKind,
had_line_break: bool,
had_line_break_before_last: bool,
is_expr_allowed: bool,
) -> bool {
let State {
ref mut context,
had_line_break,
had_line_break_before_last,
is_expr_allowed,
syntax,
..
} = *self;

let is_next_keyword = matches!(next, TokenKind::Word(WordKind::Keyword(..)));

if is_next_keyword && prev == Some(TokenType::Dot) {
Expand Down Expand Up @@ -612,7 +608,8 @@ impl State {
if let Some(TokenContext::Tpl { .. }) = context.current() {
context.pop();
} else {
context.push(TokenContext::Tpl { start });
self.tpl_start = start;
context.push(TokenContext::Tpl);
}
false
}
Expand Down Expand Up @@ -761,7 +758,6 @@ impl TokenContexts {
/// The algorithm used to determine whether a regexp can appear at a
/// given point in the program is loosely based on sweet.js' approach.
/// See https://github.com/mozilla/sweet.js/wiki/design
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenContext {
BraceStmt,
Expand All @@ -772,10 +768,7 @@ pub enum TokenContext {
is_for_loop: bool,
},
ParenExpr,
Tpl {
/// Start of a template literal.
start: BytePos,
},
Tpl,
FnExpr,
ClassExpr,
JSXOpeningTag,
Expand Down
42 changes: 21 additions & 21 deletions crates/swc_ecma_parser/src/lexer/table.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ const ERR: ByteHandler = Some(|lexer| {
const IDN: ByteHandler = Some(|lexer| lexer.read_ident_unknown().map(Some));

const L_A: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"abstract" => Some(Word::Ident(IdentLike::Known(KnownIdent::Abstract))),
"as" => Some(Word::Ident(IdentLike::Known(KnownIdent::As))),
"await" => Some(Word::Keyword(Keyword::Await)),
Expand All @@ -77,7 +77,7 @@ const L_A: ByteHandler = Some(|lexer| {
});

const L_B: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"break" => Some(Word::Keyword(Keyword::Break)),
"boolean" => Some(Word::Ident(IdentLike::Known(KnownIdent::Boolean))),
"bigint" => Some(Word::Ident(IdentLike::Known(KnownIdent::Bigint))),
Expand All @@ -86,7 +86,7 @@ const L_B: ByteHandler = Some(|lexer| {
});

const L_C: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"case" => Some(Word::Keyword(Keyword::Case)),
"catch" => Some(Word::Keyword(Keyword::Catch)),
"class" => Some(Word::Keyword(Keyword::Class)),
Expand All @@ -97,7 +97,7 @@ const L_C: ByteHandler = Some(|lexer| {
});

const L_D: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"debugger" => Some(Word::Keyword(Keyword::Debugger)),
"default" => Some(Word::Keyword(Keyword::Default_)),
"delete" => Some(Word::Keyword(Keyword::Delete)),
Expand All @@ -108,7 +108,7 @@ const L_D: ByteHandler = Some(|lexer| {
});

const L_E: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"else" => Some(Word::Keyword(Keyword::Else)),
"enum" => Some(Word::Ident(IdentLike::Known(KnownIdent::Enum))),
"export" => Some(Word::Keyword(Keyword::Export)),
Expand All @@ -118,7 +118,7 @@ const L_E: ByteHandler = Some(|lexer| {
});

const L_F: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"false" => Some(Word::False),
"finally" => Some(Word::Keyword(Keyword::Finally)),
"for" => Some(Word::Keyword(Keyword::For)),
Expand All @@ -129,7 +129,7 @@ const L_F: ByteHandler = Some(|lexer| {
});

const L_G: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"global" => Some(Word::Ident(IdentLike::Known(KnownIdent::Global))),
"get" => Some(Word::Ident(IdentLike::Known(KnownIdent::Get))),
_ => None,
Expand All @@ -139,7 +139,7 @@ const L_G: ByteHandler = Some(|lexer| {
const L_H: ByteHandler = IDN;

const L_I: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"if" => Some(Word::Keyword(Keyword::If)),
"import" => Some(Word::Keyword(Keyword::Import)),
"in" => Some(Word::Keyword(Keyword::In)),
Expand All @@ -156,28 +156,28 @@ const L_I: ByteHandler = Some(|lexer| {
const L_J: ByteHandler = IDN;

const L_K: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"keyof" => Some(Word::Ident(IdentLike::Known(KnownIdent::Keyof))),
_ => None,
})
});

const L_L: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"let" => Some(Word::Keyword(Keyword::Let)),
_ => None,
})
});

const L_M: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"meta" => Some(Word::Ident(IdentLike::Known(KnownIdent::Meta))),
_ => None,
})
});

const L_N: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"new" => Some(Word::Keyword(Keyword::New)),
"null" => Some(Word::Null),
"number" => Some(Word::Ident(IdentLike::Known(KnownIdent::Number))),
Expand All @@ -188,15 +188,15 @@ const L_N: ByteHandler = Some(|lexer| {
});

const L_O: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"of" => Some(Word::Ident(IdentLike::Known(KnownIdent::Of))),
"object" => Some(Word::Ident(IdentLike::Known(KnownIdent::Object))),
_ => None,
})
});

const L_P: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"public" => Some(Word::Ident(IdentLike::Known(KnownIdent::Public))),
"package" => Some(Word::Ident(IdentLike::Known(KnownIdent::Package))),
"protected" => Some(Word::Ident(IdentLike::Known(KnownIdent::Protected))),
Expand All @@ -208,7 +208,7 @@ const L_P: ByteHandler = Some(|lexer| {
const L_Q: ByteHandler = IDN;

const L_R: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"return" => Some(Word::Keyword(Keyword::Return)),
"readonly" => Some(Word::Ident(IdentLike::Known(KnownIdent::Readonly))),
"require" => Some(Word::Ident(IdentLike::Known(KnownIdent::Require))),
Expand All @@ -217,7 +217,7 @@ const L_R: ByteHandler = Some(|lexer| {
});

const L_S: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"super" => Some(Word::Keyword(Keyword::Super)),
"static" => Some(Word::Ident(IdentLike::Known(KnownIdent::Static))),
"switch" => Some(Word::Keyword(Keyword::Switch)),
Expand All @@ -230,7 +230,7 @@ const L_S: ByteHandler = Some(|lexer| {
});

const L_T: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"this" => Some(Word::Keyword(Keyword::This)),
"throw" => Some(Word::Keyword(Keyword::Throw)),
"true" => Some(Word::True),
Expand All @@ -243,7 +243,7 @@ const L_T: ByteHandler = Some(|lexer| {
});

const L_U: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"using" => Some(Word::Ident(IdentLike::Known(KnownIdent::Using))),
"unique" => Some(Word::Ident(IdentLike::Known(KnownIdent::Unique))),
"undefined" => Some(Word::Ident(IdentLike::Known(KnownIdent::Undefined))),
Expand All @@ -253,15 +253,15 @@ const L_U: ByteHandler = Some(|lexer| {
});

const L_V: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"var" => Some(Word::Keyword(Keyword::Var)),
"void" => Some(Word::Keyword(Keyword::Void)),
_ => None,
})
});

const L_W: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"while" => Some(Word::Keyword(Keyword::While)),
"with" => Some(Word::Keyword(Keyword::With)),
_ => None,
Expand All @@ -271,7 +271,7 @@ const L_W: ByteHandler = Some(|lexer| {
const L_X: ByteHandler = IDN;

const L_Y: ByteHandler = Some(|lexer| {
lexer.read_word_with(|s| match s {
lexer.read_word_with(&|s| match s {
"yield" => Some(Word::Keyword(Keyword::Yield)),
_ => None,
})
Expand Down
Loading