Consider soft keyword for expr / pattern parsing

astral-sh · May 20, 2024 · 0fbe59f · 0fbe59f
1 parent 56ee76e
commit 0fbe59f
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 11 deletions.
diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs
@@ -35,9 +35,12 @@ const LITERAL_SET: TokenSet = TokenSet::new([
     TokenKind::None,
 ]);
 
+/// Tokens that represent a soft keyword.
+pub(super) const SOFT_KEYWORD_SET: TokenSet =
+    TokenSet::new([TokenKind::Case, TokenKind::Match, TokenKind::Type]);
+
 /// Tokens that represents either an expression or the start of one.
 pub(super) const EXPR_SET: TokenSet = TokenSet::new([
-    TokenKind::Case,
     TokenKind::Name,
     TokenKind::Minus,
     TokenKind::Plus,
@@ -54,7 +57,8 @@ pub(super) const EXPR_SET: TokenSet = TokenSet::new([
     TokenKind::FStringStart,
     TokenKind::IpyEscapeCommand,
 ])
-.union(LITERAL_SET);
+.union(LITERAL_SET)
+.union(SOFT_KEYWORD_SET);
 
 /// Tokens that can appear after an expression.
 pub(super) const END_EXPR_SET: TokenSet = TokenSet::new([
@@ -108,6 +112,11 @@ pub(super) const END_EXPR_SET: TokenSet = TokenSet::new([
 const END_SEQUENCE_SET: TokenSet = END_EXPR_SET.remove(TokenKind::Comma);
 
 impl<'src> Parser<'src> {
+    /// Returns `true` if the current token is a soft keyword.
+    pub(super) fn at_soft_keyword(&self) -> bool {
+        self.at_ts(SOFT_KEYWORD_SET)
+    }
+
     /// Returns `true` if the current token is the start of an expression.
     pub(super) fn at_expr(&self) -> bool {
         self.at_ts(EXPR_SET)
@@ -1434,6 +1443,8 @@ impl<'src> Parser<'src> {
                     ParseErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
                     conversion_flag_range,
                 );
+                // TODO(dhruvmanila): Avoid dropping this token
+                self.bump_any();
                 ConversionFlag::None
             }
         } else {

diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs
@@ -418,7 +418,7 @@ impl<'src> Parser<'src> {
     ///
     /// If the current token is not a soft keyword.
     pub(crate) fn bump_soft_keyword_as_name(&mut self) {
-        assert!(self.current_token_kind().is_soft_keyword());
+        assert!(self.at_soft_keyword());
 
         self.do_bump(TokenKind::Name);
     }
@@ -1050,9 +1050,10 @@ impl RecoveryContextKind {
             RecoveryContextKind::Except => p.at(TokenKind::Except),
             RecoveryContextKind::AssignmentTargets => p.at(TokenKind::Equal),
             RecoveryContextKind::TypeParams => p.at_type_param(),
-            RecoveryContextKind::ImportNames => p.at(TokenKind::Name),
+            RecoveryContextKind::ImportNames => p.at(TokenKind::Name) || p.at_soft_keyword(),
             RecoveryContextKind::ImportFromAsNames(_) => {
                 matches!(p.current_token_kind(), TokenKind::Star | TokenKind::Name)
+                    || p.at_soft_keyword()
             }
             RecoveryContextKind::Slices => p.at(TokenKind::Colon) || p.at_expr(),
             RecoveryContextKind::ListElements
@@ -1071,11 +1072,13 @@ impl RecoveryContextKind {
             RecoveryContextKind::MatchPatternClassArguments => p.at_pattern_start(),
             RecoveryContextKind::Arguments => p.at_expr(),
             RecoveryContextKind::DeleteTargets => p.at_expr(),
-            RecoveryContextKind::Identifiers => p.at(TokenKind::Name),
-            RecoveryContextKind::Parameters(_) => matches!(
-                p.current_token_kind(),
-                TokenKind::Name | TokenKind::Star | TokenKind::DoubleStar | TokenKind::Slash
-            ),
+            RecoveryContextKind::Identifiers => p.at(TokenKind::Name) || p.at_soft_keyword(),
+            RecoveryContextKind::Parameters(_) => {
+                matches!(
+                    p.current_token_kind(),
+                    TokenKind::Name | TokenKind::Star | TokenKind::DoubleStar | TokenKind::Slash
+                ) || p.at_soft_keyword()
+            }
             RecoveryContextKind::WithItems(_) => p.at_expr(),
             RecoveryContextKind::FStringElements => matches!(
                 p.current_token_kind(),

diff --git a/crates/ruff_python_parser/src/parser/pattern.rs b/crates/ruff_python_parser/src/parser/pattern.rs
@@ -2,6 +2,7 @@ use ruff_python_ast::{self as ast, Expr, ExprContext, Number, Operator, Pattern,
 use ruff_text_size::{Ranged, TextSize};
 
 use crate::lexer::TokenValue;
+use crate::parser::expression::SOFT_KEYWORD_SET;
 use crate::parser::progress::ParserProgress;
 use crate::parser::{recovery, Parser, RecoveryContextKind, SequenceMatchPatternParentheses};
 use crate::token_set::TokenSet;
@@ -37,7 +38,8 @@ const PATTERN_START_SET: TokenSet = TokenSet::new([
     // Mapping pattern
     TokenKind::Lbrace,
 ])
-.union(LITERAL_PATTERN_START_SET);
+.union(LITERAL_PATTERN_START_SET)
+.union(SOFT_KEYWORD_SET);
 
 /// The set of tokens that can start a mapping pattern.
 const MAPPING_PATTERN_START_SET: TokenSet = TokenSet::new([
@@ -46,7 +48,8 @@ const MAPPING_PATTERN_START_SET: TokenSet = TokenSet::new([
     // Value pattern
     TokenKind::Name,
 ])
-.union(LITERAL_PATTERN_START_SET);
+.union(LITERAL_PATTERN_START_SET)
+.union(SOFT_KEYWORD_SET);
 
 impl<'src> Parser<'src> {
     /// Returns `true` if the current token is a valid start of a pattern.