Use struct fields in TokenValue enum (#11475)

## Summary This PR updates certain `TokenValue` enum variants to use struct fields instead of tuple variants. The main reason is to avoid a large diff for test snapshots, so it becomes easier to diagnose any issues. This is temporary and will be updated once everything is finalized.
astral-sh · May 30, 2024 · 78e5b47 · 78e5b47
1 parent f35cc56
commit 78e5b47
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 17 deletions.
diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs
@@ -195,7 +195,9 @@ impl<'src> Lexer<'src> {
         let text = self.token_text();
 
         if !is_ascii {
-            self.value = TokenValue::Name(text.nfkc().collect::<String>().into_boxed_str());
+            self.value = TokenValue::Name {
+                name: text.nfkc().collect::<String>().into_boxed_str(),
+            };
             return TokenKind::Name;
         }
 
@@ -239,7 +241,9 @@ impl<'src> Lexer<'src> {
             "with" => TokenKind::With,
             "yield" => TokenKind::Yield,
             _ => {
-                self.value = TokenValue::Name(text.to_string().into_boxed_str());
+                self.value = TokenValue::Name {
+                    name: text.to_string().into_boxed_str(),
+                };
                 TokenKind::Name
             }
         }
@@ -286,7 +290,7 @@ impl<'src> Lexer<'src> {
                 ));
             }
         };
-        self.value = TokenValue::Int(value);
+        self.value = TokenValue::Int { value };
         TokenKind::Int
     }
 
@@ -354,7 +358,7 @@ impl<'src> Lexer<'src> {
                 };
                 TokenKind::Complex
             } else {
-                self.value = TokenValue::Float(value);
+                self.value = TokenValue::Float { value };
                 TokenKind::Float
             }
         } else {
@@ -386,7 +390,7 @@ impl<'src> Lexer<'src> {
                         ))
                     }
                 };
-                self.value = TokenValue::Int(value);
+                self.value = TokenValue::Int { value };
                 TokenKind::Int
             }
         }
@@ -1516,14 +1520,23 @@ pub(crate) enum TokenValue {
     #[default]
     None,
     /// Token value for a name, commonly known as an identifier.
-    ///
-    /// Unicode names are NFKC-normalized by the lexer, matching
-    /// [the behaviour of Python's lexer](https://docs.python.org/3/reference/lexical_analysis.html#identifiers)
-    Name(Box<str>),
+    Name {
+        /// The name value.
+        ///
+        /// Unicode names are NFKC-normalized by the lexer,
+        /// matching [the behaviour of Python's lexer](https://docs.python.org/3/reference/lexical_analysis.html#identifiers)
+        name: Box<str>,
+    },
     /// Token value for an integer.
-    Int(Int),
+    Int {
+        /// The integer value.
+        value: Int,
+    },
     /// Token value for a floating point number.
-    Float(f64),
+    Float {
+        /// The float value.
+        value: f64,
+    },
     /// Token value for a complex number.
     Complex {
         /// The real part of the complex number.

diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs
@@ -475,7 +475,7 @@ impl<'src> Parser<'src> {
         let range = self.current_token_range();
 
         if self.at(TokenKind::Name) {
-            let TokenValue::Name(name) = self.bump_value(TokenKind::Name) else {
+            let TokenValue::Name { name } = self.bump_value(TokenKind::Name) else {
                 unreachable!();
             };
             return ast::Identifier {
@@ -524,7 +524,7 @@ impl<'src> Parser<'src> {
 
         let lhs = match self.current_token_kind() {
             TokenKind::Float => {
-                let TokenValue::Float(value) = self.bump_value(TokenKind::Float) else {
+                let TokenValue::Float { value } = self.bump_value(TokenKind::Float) else {
                     unreachable!()
                 };
 
@@ -543,7 +543,7 @@ impl<'src> Parser<'src> {
                 })
             }
             TokenKind::Int => {
-                let TokenValue::Int(value) = self.bump_value(TokenKind::Int) else {
+                let TokenValue::Int { value } = self.bump_value(TokenKind::Int) else {
                     unreachable!()
                 };
                 Expr::NumberLiteral(ast::ExprNumberLiteral {
@@ -1423,7 +1423,7 @@ impl<'src> Parser<'src> {
         let conversion = if self.eat(TokenKind::Exclamation) {
             let conversion_flag_range = self.current_token_range();
             if self.at(TokenKind::Name) {
-                let TokenValue::Name(name) = self.bump_value(TokenKind::Name) else {
+                let TokenValue::Name { name } = self.bump_value(TokenKind::Name) else {
                     unreachable!();
                 };
                 match &*name {

diff --git a/crates/ruff_python_parser/src/parser/pattern.rs b/crates/ruff_python_parser/src/parser/pattern.rs
@@ -412,7 +412,7 @@ impl<'src> Parser<'src> {
                 })
             }
             TokenKind::Int => {
-                let TokenValue::Int(value) = self.bump_value(TokenKind::Int) else {
+                let TokenValue::Int { value } = self.bump_value(TokenKind::Int) else {
                     unreachable!()
                 };
                 let range = self.node_range(start);
@@ -426,7 +426,7 @@ impl<'src> Parser<'src> {
                 })
             }
             TokenKind::Float => {
-                let TokenValue::Float(value) = self.bump_value(TokenKind::Float) else {
+                let TokenValue::Float { value } = self.bump_value(TokenKind::Float) else {
                     unreachable!()
                 };
                 let range = self.node_range(start);