Skip to content

Commit

Permalink
Use correct range to highlight line continuation error (#12016)
Browse files Browse the repository at this point in the history
## Summary

This PR fixes the range highlighted for the line continuation error.

Previously, it would highlight an incorrect range:
```
1 | call(a, b, \\\
  |           ^^ Syntax Error: unexpected character after line continuation character
2 | 
3 | def bar():
  |
```

And now:
```
  |
1 | call(a, b, \\\
  |             ^ Syntax Error: unexpected character after line continuation character
2 | 
3 | def bar():
  |
```

This is implemented by avoiding to update the token range for the
`Unknown` token which is emitted when there's a lexical error. Instead,
the `push_error` helper method will be responsible to update the range
to the error location.

This actually becomes a requirement which can be seen in follow-up PRs.

## Test Plan

Update and validate the snapshot.
  • Loading branch information
dhruvmanila authored Jun 25, 2024
1 parent 692309e commit 9c1b6ec
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 14 deletions.
23 changes: 14 additions & 9 deletions crates/ruff_python_parser/src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,24 @@ impl<'src> Lexer<'src> {
std::mem::take(&mut self.current_value)
}

/// Helper function to push the given error, updating the current range with the error location
/// and return the [`TokenKind::Unknown`] token.
fn push_error(&mut self, error: LexicalError) -> TokenKind {
self.current_range = error.location();
self.errors.push(error);
TokenKind::Unknown
}

/// Lex the next token.
pub fn next_token(&mut self) -> TokenKind {
self.cursor.start_token();
self.current_value = TokenValue::None;
self.current_flags = TokenFlags::empty();
self.current_kind = self.lex_token();
self.current_range = self.token_range();
// For `Unknown` token, the `push_error` method updates the current range.
if !matches!(self.current_kind, TokenKind::Unknown) {
self.current_range = self.token_range();
}
self.current_kind
}

Expand Down Expand Up @@ -236,7 +247,7 @@ impl<'src> Lexer<'src> {
} else if !self.cursor.eat_char('\n') {
return Some(self.push_error(LexicalError::new(
LexicalErrorType::LineContinuationError,
self.token_range(),
TextRange::at(self.offset(), self.cursor.first().text_len()),
)));
}
indentation = Indentation::root();
Expand Down Expand Up @@ -328,7 +339,7 @@ impl<'src> Lexer<'src> {
} else if !self.cursor.eat_char('\n') {
return Err(LexicalError::new(
LexicalErrorType::LineContinuationError,
self.token_range(),
TextRange::at(self.offset(), self.cursor.first().text_len()),
));
}
}
Expand Down Expand Up @@ -1464,12 +1475,6 @@ impl<'src> Lexer<'src> {
self.token_range().start()
}

/// Helper function to push the given error and return the [`TokenKind::Unknown`] token.
fn push_error(&mut self, error: LexicalError) -> TokenKind {
self.errors.push(error);
TokenKind::Unknown
}

/// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`].
pub(crate) fn checkpoint(&self) -> LexerCheckpoint {
LexerCheckpoint {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ Module(
body: [
Expr(
StmtExpr {
range: 0..13,
range: 0..14,
value: Call(
ExprCall {
range: 0..13,
range: 0..14,
func: Name(
ExprName {
range: 0..4,
Expand All @@ -23,7 +23,7 @@ Module(
},
),
arguments: Arguments {
range: 4..13,
range: 4..14,
args: [
Name(
ExprName {
Expand Down Expand Up @@ -82,15 +82,15 @@ Module(

|
1 | call(a, b, \\\
| ^^ Syntax Error: unexpected character after line continuation character
| ^ Syntax Error: unexpected character after line continuation character
2 |
3 | def bar():
|


|
1 | call(a, b, \\\
| ^ Syntax Error: unexpected character after line continuation character
| ^ Syntax Error: unexpected character after line continuation character
2 |
3 | def bar():
|
Expand Down

0 comments on commit 9c1b6ec

Please sign in to comment.