diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 9d202e4e6e2213..3de0d218205d70 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -195,7 +195,9 @@ impl<'src> Lexer<'src> { let text = self.token_text(); if !is_ascii { - self.value = TokenValue::Name(text.nfkc().collect::().into_boxed_str()); + self.value = TokenValue::Name { + name: text.nfkc().collect::().into_boxed_str(), + }; return TokenKind::Name; } @@ -239,7 +241,9 @@ impl<'src> Lexer<'src> { "with" => TokenKind::With, "yield" => TokenKind::Yield, _ => { - self.value = TokenValue::Name(text.to_string().into_boxed_str()); + self.value = TokenValue::Name { + name: text.to_string().into_boxed_str(), + }; TokenKind::Name } } @@ -286,7 +290,7 @@ impl<'src> Lexer<'src> { )); } }; - self.value = TokenValue::Int(value); + self.value = TokenValue::Int { value }; TokenKind::Int } @@ -354,7 +358,7 @@ impl<'src> Lexer<'src> { }; TokenKind::Complex } else { - self.value = TokenValue::Float(value); + self.value = TokenValue::Float { value }; TokenKind::Float } } else { @@ -386,7 +390,7 @@ impl<'src> Lexer<'src> { )) } }; - self.value = TokenValue::Int(value); + self.value = TokenValue::Int { value }; TokenKind::Int } } @@ -1516,14 +1520,23 @@ pub(crate) enum TokenValue { #[default] None, /// Token value for a name, commonly known as an identifier. - /// - /// Unicode names are NFKC-normalized by the lexer, matching - /// [the behaviour of Python's lexer](https://docs.python.org/3/reference/lexical_analysis.html#identifiers) - Name(Box), + Name { + /// The name value. + /// + /// Unicode names are NFKC-normalized by the lexer, + /// matching [the behaviour of Python's lexer](https://docs.python.org/3/reference/lexical_analysis.html#identifiers) + name: Box, + }, /// Token value for an integer. - Int(Int), + Int { + /// The integer value. + value: Int, + }, /// Token value for a floating point number. - Float(f64), + Float { + /// The float value. + value: f64, + }, /// Token value for a complex number. Complex { /// The real part of the complex number. diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index c93782478f478f..d82b7d7cf682f8 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -470,7 +470,7 @@ impl<'src> Parser<'src> { let range = self.current_token_range(); if self.at(TokenKind::Name) { - let TokenValue::Name(name) = self.bump_value(TokenKind::Name) else { + let TokenValue::Name { name } = self.bump_value(TokenKind::Name) else { unreachable!(); }; return ast::Identifier { @@ -519,7 +519,7 @@ impl<'src> Parser<'src> { let lhs = match self.current_token_kind() { TokenKind::Float => { - let TokenValue::Float(value) = self.bump_value(TokenKind::Float) else { + let TokenValue::Float { value } = self.bump_value(TokenKind::Float) else { unreachable!() }; @@ -538,7 +538,7 @@ impl<'src> Parser<'src> { }) } TokenKind::Int => { - let TokenValue::Int(value) = self.bump_value(TokenKind::Int) else { + let TokenValue::Int { value } = self.bump_value(TokenKind::Int) else { unreachable!() }; Expr::NumberLiteral(ast::ExprNumberLiteral { @@ -1418,7 +1418,7 @@ impl<'src> Parser<'src> { let conversion = if self.eat(TokenKind::Exclamation) { let conversion_flag_range = self.current_token_range(); if self.at(TokenKind::Name) { - let TokenValue::Name(name) = self.bump_value(TokenKind::Name) else { + let TokenValue::Name { name } = self.bump_value(TokenKind::Name) else { unreachable!(); }; match &*name { diff --git a/crates/ruff_python_parser/src/parser/pattern.rs b/crates/ruff_python_parser/src/parser/pattern.rs index 2c3534fcbb98a5..7b484be3447b36 100644 --- a/crates/ruff_python_parser/src/parser/pattern.rs +++ b/crates/ruff_python_parser/src/parser/pattern.rs @@ -415,7 +415,7 @@ impl<'src> Parser<'src> { }) } TokenKind::Int => { - let TokenValue::Int(value) = self.bump_value(TokenKind::Int) else { + let TokenValue::Int { value } = self.bump_value(TokenKind::Int) else { unreachable!() }; let range = self.node_range(start); @@ -429,7 +429,7 @@ impl<'src> Parser<'src> { }) } TokenKind::Float => { - let TokenValue::Float(value) = self.bump_value(TokenKind::Float) else { + let TokenValue::Float { value } = self.bump_value(TokenKind::Float) else { unreachable!() }; let range = self.node_range(start); @@ -443,7 +443,7 @@ impl<'src> Parser<'src> { }) } TokenKind::Name if self.peek() == TokenKind::Dot => { - let TokenValue::Name(name) = self.bump_value(TokenKind::Name) else { + let TokenValue::Name { name } = self.bump_value(TokenKind::Name) else { unreachable!() }; let id = Expr::Name(ast::ExprName { @@ -460,7 +460,7 @@ impl<'src> Parser<'src> { }) } TokenKind::Name => { - let TokenValue::Name(name) = self.bump_value(TokenKind::Name) else { + let TokenValue::Name { name } = self.bump_value(TokenKind::Name) else { unreachable!() }; let range = self.node_range(start);