diff --git a/Cargo.lock b/Cargo.lock index b28fcd93ad867..d4f54ad8aae12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2014,6 +2014,7 @@ dependencies = [ "ruff_python_parser", "ruff_python_stdlib", "ruff_python_trivia", + "ruff_text_size", "ruff_workspace", "schemars", "serde", @@ -2190,6 +2191,7 @@ dependencies = [ "ruff_python_literal", "ruff_python_parser", "ruff_source_file", + "ruff_text_size", ] [[package]] diff --git a/crates/red_knot/src/parse.rs b/crates/red_knot/src/parse.rs index 6856315dcb494..4e3cd4d422119 100644 --- a/crates/red_knot/src/parse.rs +++ b/crates/red_knot/src/parse.rs @@ -32,17 +32,19 @@ impl Parsed { let result = ruff_python_parser::parse(text, Mode::Module); let (module, errors) = match result { - Ok(ast::Mod::Module(module)) => (module, vec![]), - Ok(ast::Mod::Expression(expression)) => ( - ast::ModModule { - range: expression.range(), - body: vec![ast::Stmt::Expr(ast::StmtExpr { + Ok(parsed) => match parsed.into_syntax() { + ast::Mod::Module(module) => (module, vec![]), + ast::Mod::Expression(expression) => ( + ast::ModModule { range: expression.range(), - value: expression.body, - })], - }, - vec![], - ), + body: vec![ast::Stmt::Expr(ast::StmtExpr { + range: expression.range(), + value: expression.body, + })], + }, + vec![], + ), + }, Err(errors) => ( ast::ModModule { range: TextRange::default(), diff --git a/crates/ruff_benchmark/benches/lexer.rs b/crates/ruff_benchmark/benches/lexer.rs index c31cb84b5e4f2..b8b464bfbb4ee 100644 --- a/crates/ruff_benchmark/benches/lexer.rs +++ b/crates/ruff_benchmark/benches/lexer.rs @@ -2,7 +2,7 @@ use ruff_benchmark::criterion::{ criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput, }; use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError}; -use ruff_python_parser::{lexer, Mode}; +use ruff_python_parser::{lexer, Mode, TokenKind}; #[cfg(target_os = "windows")] #[global_allocator] @@ -47,9 +47,15 @@ fn benchmark_lexer(criterion: &mut Criterion) { &case, |b, case| { b.iter(|| { - let result = - lexer::lex(case.code(), Mode::Module).find(std::result::Result::is_err); - assert_eq!(result, None, "Input to be a valid Python program"); + let mut lexer = lexer::lex(case.code(), Mode::Module); + loop { + let token = lexer.next_token(); + match token { + TokenKind::EndOfFile => break, + TokenKind::Unknown => panic!("Input to be a valid Python program"), + _ => {} + } + } }); }, ); diff --git a/crates/ruff_benchmark/benches/linter.rs b/crates/ruff_benchmark/benches/linter.rs index 77ccf03e4097c..07c235be04fb5 100644 --- a/crates/ruff_benchmark/benches/linter.rs +++ b/crates/ruff_benchmark/benches/linter.rs @@ -10,7 +10,7 @@ use ruff_linter::settings::{flags, LinterSettings}; use ruff_linter::source_kind::SourceKind; use ruff_linter::{registry::Rule, RuleSelector}; use ruff_python_ast::PySourceType; -use ruff_python_parser::{parse_module, Mode}; +use ruff_python_parser::parse_module; #[cfg(target_os = "windows")] #[global_allocator] diff --git a/crates/ruff_benchmark/benches/parser.rs b/crates/ruff_benchmark/benches/parser.rs index 3c24c8144178b..ec2fa671c1df0 100644 --- a/crates/ruff_benchmark/benches/parser.rs +++ b/crates/ruff_benchmark/benches/parser.rs @@ -4,7 +4,7 @@ use ruff_benchmark::criterion::{ use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError}; use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor}; use ruff_python_ast::Stmt; -use ruff_python_parser::{parse_module, Mode}; +use ruff_python_parser::parse_module; #[cfg(target_os = "windows")] #[global_allocator] diff --git a/crates/ruff_dev/Cargo.toml b/crates/ruff_dev/Cargo.toml index 632c12f473786..d5ccc937fd9c5 100644 --- a/crates/ruff_dev/Cargo.toml +++ b/crates/ruff_dev/Cargo.toml @@ -22,6 +22,7 @@ ruff_python_formatter = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } ruff_python_trivia = { workspace = true } +ruff_text_size = { workspace = true } ruff_workspace = { workspace = true, features = ["schemars"] } anyhow = { workspace = true } diff --git a/crates/ruff_dev/src/print_tokens.rs b/crates/ruff_dev/src/print_tokens.rs index 09e172b7b5a2b..0f6e3dbddda0a 100644 --- a/crates/ruff_dev/src/print_tokens.rs +++ b/crates/ruff_dev/src/print_tokens.rs @@ -8,6 +8,7 @@ use anyhow::Result; use ruff_linter::source_kind::SourceKind; use ruff_python_ast::PySourceType; use ruff_python_parser::parse_unchecked_source; +use ruff_text_size::Ranged; #[derive(clap::Args)] pub(crate) struct Args { @@ -29,8 +30,8 @@ pub(crate) fn main(args: &Args) -> Result<()> { println!( "{start:#?} {kind:#?} {end:#?}", start = token.start(), - end = token.end() - kind = token.kind() + end = token.end(), + kind = token.kind(), ); } Ok(()) diff --git a/crates/ruff_linter/src/checkers/imports.rs b/crates/ruff_linter/src/checkers/imports.rs index 5d24ff59767e1..b2abaef2195c2 100644 --- a/crates/ruff_linter/src/checkers/imports.rs +++ b/crates/ruff_linter/src/checkers/imports.rs @@ -4,10 +4,10 @@ use std::path::Path; use ruff_diagnostics::Diagnostic; use ruff_notebook::CellOffsets; use ruff_python_ast::statement_visitor::StatementVisitor; -use ruff_python_ast::{ModModule, PySourceType, Suite}; +use ruff_python_ast::{ModModule, PySourceType}; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::{Program, Tokens}; +use ruff_python_parser::Program; use ruff_source_file::Locator; use crate::directives::IsortDirectives; diff --git a/crates/ruff_linter/src/checkers/logical_lines.rs b/crates/ruff_linter/src/checkers/logical_lines.rs index 4044e6c18a67b..ef9a7a8dae8e5 100644 --- a/crates/ruff_linter/src/checkers/logical_lines.rs +++ b/crates/ruff_linter/src/checkers/logical_lines.rs @@ -2,8 +2,7 @@ use crate::line_width::IndentWidth; use ruff_diagnostics::Diagnostic; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::lexer::LexResult; -use ruff_python_parser::TokenKind; +use ruff_python_parser::{TokenKind, Tokens}; use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextRange}; @@ -34,7 +33,7 @@ pub(crate) fn expand_indent(line: &str, indent_width: IndentWidth) -> usize { } pub(crate) fn check_logical_lines( - tokens: &[LexResult], + tokens: &Tokens, locator: &Locator, indexer: &Indexer, stylist: &Stylist, diff --git a/crates/ruff_linter/src/checkers/tokens.rs b/crates/ruff_linter/src/checkers/tokens.rs index 033f8fb447c2f..fae8ce0928cd5 100644 --- a/crates/ruff_linter/src/checkers/tokens.rs +++ b/crates/ruff_linter/src/checkers/tokens.rs @@ -8,7 +8,7 @@ use ruff_python_codegen::Stylist; use ruff_diagnostics::Diagnostic; use ruff_python_index::Indexer; -use ruff_python_parser::{Program, Tokens}; +use ruff_python_parser::Program; use ruff_source_file::Locator; use ruff_text_size::Ranged; diff --git a/crates/ruff_linter/src/directives.rs b/crates/ruff_linter/src/directives.rs index 5d75799c6482c..37e346da52f1f 100644 --- a/crates/ruff_linter/src/directives.rs +++ b/crates/ruff_linter/src/directives.rs @@ -4,9 +4,8 @@ use std::iter::Peekable; use std::str::FromStr; use bitflags::bitflags; -use ruff_python_ast::{ModModule, StringFlags}; -use ruff_python_parser::lexer::LexResult; -use ruff_python_parser::{Program, Tok, TokenKind, Tokens}; +use ruff_python_ast::ModModule; +use ruff_python_parser::{Program, TokenKind, Tokens}; use ruff_python_trivia::CommentRanges; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; @@ -60,7 +59,7 @@ pub fn extract_directives( ) -> Directives { Directives { noqa_line_for: if flags.intersects(Flags::NOQA) { - extract_noqa_line_for(lxr, locator, indexer) + extract_noqa_line_for(program.tokens(), locator, indexer) } else { NoqaMapping::default() }, @@ -380,8 +379,7 @@ impl TodoDirectiveKind { #[cfg(test)] mod tests { - use ruff_python_parser::lexer::LexResult; - use ruff_python_parser::{lexer, parse_module, Mode}; + use ruff_python_parser::parse_module; use ruff_text_size::{TextLen, TextRange, TextSize}; use ruff_python_index::Indexer; diff --git a/crates/ruff_linter/src/fix/edits.rs b/crates/ruff_linter/src/fix/edits.rs index 34026e3fb6fc6..1746dc0e7d14c 100644 --- a/crates/ruff_linter/src/fix/edits.rs +++ b/crates/ruff_linter/src/fix/edits.rs @@ -533,7 +533,7 @@ mod tests { use ruff_diagnostics::{Diagnostic, Edit, Fix}; use ruff_python_ast::Stmt; use ruff_python_codegen::Stylist; - use ruff_python_parser::{lexer, parse, parse_expression, parse_module, Mode}; + use ruff_python_parser::{parse_expression, parse_module}; use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -615,7 +615,7 @@ x = 1 \ } #[test] - fn redundant_alias() { + fn redundant_alias() -> Result<()> { let contents = "import x, y as y, z as bees"; let stmt = parse_first_stmt(contents)?; assert_eq!( @@ -642,6 +642,7 @@ x = 1 \ )], "the third item is already aliased to something else" ); + Ok(()) } #[test_case("()", &["x", "y"], r#"("x", "y")"# ; "2 into empty tuple")] diff --git a/crates/ruff_linter/src/importer/insertion.rs b/crates/ruff_linter/src/importer/insertion.rs index 5805e3e0009de..70b160a683b2f 100644 --- a/crates/ruff_linter/src/importer/insertion.rs +++ b/crates/ruff_linter/src/importer/insertion.rs @@ -1,8 +1,8 @@ //! Insert statements into Python code. use std::ops::Add; -use ruff_python_ast::{PySourceType, Stmt}; -use ruff_python_parser::{lexer, AsMode, Tok, TokenKind, Tokens}; +use ruff_python_ast::Stmt; +use ruff_python_parser::{TokenKind, Tokens}; use ruff_text_size::{Ranged, TextSize}; use ruff_diagnostics::Edit; @@ -172,7 +172,7 @@ impl<'a> Insertion<'a> { // Once we've seen the colon, we're looking for a newline; otherwise, there's no // block body (e.g. `if True: pass`). Awaiting::Newline => match token.kind() { - TokenKind::Comment(..) => {} + TokenKind::Comment => {} TokenKind::Newline => { state = Awaiting::Indent; } @@ -183,7 +183,7 @@ impl<'a> Insertion<'a> { }, // Once we've seen the newline, we're looking for the indentation of the block body. Awaiting::Indent => match token.kind() { - TokenKind::Comment(..) => {} + TokenKind::Comment => {} TokenKind::NonLogicalNewline => {} TokenKind::Indent => { // This is like: @@ -317,9 +317,8 @@ fn match_continuation(s: &str) -> Option { mod tests { use anyhow::Result; - use ruff_python_ast::PySourceType; use ruff_python_codegen::Stylist; - use ruff_python_parser::{parse, parse_module, Mode}; + use ruff_python_parser::parse_module; use ruff_source_file::{LineEnding, Locator}; use ruff_text_size::TextSize; diff --git a/crates/ruff_linter/src/linter.rs b/crates/ruff_linter/src/linter.rs index eabf8942e1b00..023921753c109 100644 --- a/crates/ruff_linter/src/linter.rs +++ b/crates/ruff_linter/src/linter.rs @@ -81,18 +81,21 @@ pub fn check_path( noqa: flags::Noqa, source_kind: &SourceKind, source_type: PySourceType, - program: Program, + program: &Program, ) -> LinterResult> { // Aggregate all diagnostics. let mut diagnostics = vec![]; let mut error = None; + let tokens = program.tokens(); + let comment_ranges = program.comment_ranges(); + // Collect doc lines. This requires a rare mix of tokens (for comments) and AST // (for docstrings), which demands special-casing at this level. let use_doc_lines = settings.rules.enabled(Rule::DocLineTooLong); let mut doc_lines = vec![]; if use_doc_lines { - doc_lines.extend(doc_lines_from_tokens(program.tokens())); + doc_lines.extend(doc_lines_from_tokens(tokens)); } // Run the token-based rules. @@ -102,7 +105,7 @@ pub fn check_path( .any(|rule_code| rule_code.lint_source().is_tokens()) { diagnostics.extend(check_tokens( - &program, + program, path, locator, indexer, @@ -123,7 +126,7 @@ pub fn check_path( path, package, locator, - program.comment_ranges(), + comment_ranges, settings, )); } @@ -135,7 +138,7 @@ pub fn check_path( .any(|rule_code| rule_code.lint_source().is_logical_lines()) { diagnostics.extend(crate::checkers::logical_lines::check_logical_lines( - &program, locator, indexer, stylist, settings, + tokens, locator, indexer, stylist, settings, )); } @@ -150,13 +153,13 @@ pub fn check_path( .iter_enabled() .any(|rule_code| rule_code.lint_source().is_imports()); if use_ast || use_imports || use_doc_lines { - match program.into_result() { + match program.as_result() { Ok(program) => { let cell_offsets = source_kind.as_ipy_notebook().map(Notebook::cell_offsets); let notebook_index = source_kind.as_ipy_notebook().map(Notebook::index); if use_ast { diagnostics.extend(check_ast( - &program, + program, locator, stylist, indexer, @@ -172,7 +175,7 @@ pub fn check_path( } if use_imports { let import_diagnostics = check_imports( - &program, + program, locator, indexer, &directives.isort, @@ -195,8 +198,9 @@ pub fn check_path( // if it's disabled via any of the usual mechanisms (e.g., `noqa`, // `per-file-ignores`), and the easiest way to detect that suppression is // to see if the diagnostic persists to the end of the function. - pycodestyle::rules::syntax_error(&mut diagnostics, &parse_error, locator); - error = Some(parse_error); + pycodestyle::rules::syntax_error(&mut diagnostics, parse_error, locator); + // TODO(dhruvmanila): Remove this clone + error = Some(parse_error.clone()); } } } @@ -217,7 +221,7 @@ pub fn check_path( locator, stylist, indexer, - program.comment_ranges(), + comment_ranges, &doc_lines, settings, )); @@ -226,8 +230,6 @@ pub fn check_path( // Raise violations for internal test rules #[cfg(any(feature = "test-rules", test))] { - let comment_ranges = program.comment_ranges(); - for test_rule in TEST_RULES { if !settings.rules.enabled(*test_rule) { continue; @@ -307,7 +309,7 @@ pub fn check_path( &mut diagnostics, path, locator, - program.comment_ranges(), + comment_ranges, &directives.noqa_line_for, error.is_none(), &per_file_ignores, @@ -405,7 +407,7 @@ pub fn add_noqa_to_path( flags::Noqa::Disabled, source_kind, source_type, - program, + &program, ); // Log any parse errors. @@ -476,7 +478,7 @@ pub fn lint_only( noqa, source_kind, source_type, - program, + &program, ); result.map(|diagnostics| diagnostics_to_messages(diagnostics, path, &locator, &directives)) @@ -567,7 +569,7 @@ pub fn lint_fix<'a>( noqa, &transformed, source_type, - program, + &program, ); if iterations == 0 { diff --git a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs index 5166fbdd3eff2..00dd8d382ef56 100644 --- a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs +++ b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs @@ -139,7 +139,7 @@ pub(crate) fn unnecessary_generator_list(checker: &mut Checker, call: &ast::Expr let range = parenthesized_range( argument.into(), (&call.arguments).into(), - checker.indexer().comment_ranges(), + checker.program().comment_ranges(), checker.locator().contents(), ) .unwrap_or(argument.range()); diff --git a/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs b/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs index b7b0d15484f14..6a6e62196abcb 100644 --- a/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs +++ b/crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs @@ -4,9 +4,9 @@ use ruff_diagnostics::{AlwaysFixableViolation, Diagnostic, Fix}; use ruff_macros::{derive_message_formats, violation}; use ruff_python_ast::helpers::is_docstring_stmt; use ruff_python_ast::imports::{Alias, AnyImport, FutureImport, Import, ImportFrom}; -use ruff_python_ast::{self as ast, ModModule, PySourceType, Stmt, Suite}; +use ruff_python_ast::{self as ast, ModModule, PySourceType, Stmt}; use ruff_python_codegen::Stylist; -use ruff_python_parser::{parse_module, Program, Tokens}; +use ruff_python_parser::{parse_module, Program}; use ruff_source_file::Locator; use ruff_text_size::{TextRange, TextSize}; diff --git a/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs b/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs index 832319fa6c4c0..a4ea72803ecf8 100644 --- a/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs +++ b/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs @@ -8,7 +8,7 @@ use ruff_python_ast::whitespace::trailing_lines_end; use ruff_python_ast::{ModModule, PySourceType, Stmt}; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::{Program, Tokens}; +use ruff_python_parser::Program; use ruff_python_trivia::{leading_indentation, textwrap::indent, PythonWhitespace}; use ruff_source_file::{Locator, UniversalNewlines}; use ruff_text_size::{Ranged, TextRange}; @@ -79,7 +79,7 @@ fn matches_ignoring_indentation(val1: &str, val2: &str) -> bool { }) } -#[allow(clippy::cast_sign_loss)] +#[allow(clippy::cast_sign_loss, clippy::too_many_arguments)] /// I001 pub(crate) fn organize_imports( block: &Block, diff --git a/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs b/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs index 1fc644434c22d..463ef0a4f550f 100644 --- a/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs +++ b/crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs @@ -183,7 +183,7 @@ mod tests { use super::get_complexity_number; fn parse_suite(source: &str) -> Result { - parse_module(source).map(|program| program.into_suite()) + Ok(parse_module(source)?.into_suite()) } #[test] diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs index 1bbef536b41fb..bdfb2e9629e46 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs @@ -3,7 +3,7 @@ use std::slice::Iter; use ruff_notebook::CellOffsets; use ruff_python_ast::PySourceType; use ruff_python_parser::{Token, TokenKind, Tokens}; -use ruff_text_size::{Ranged, TextRange, TextSize}; +use ruff_text_size::{Ranged, TextSize}; use ruff_diagnostics::{AlwaysFixableViolation, Violation}; use ruff_diagnostics::{Diagnostic, Edit, Fix}; diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/doc_line_too_long.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/doc_line_too_long.rs index 37cced4082d16..5661f62036f66 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/doc_line_too_long.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/doc_line_too_long.rs @@ -1,6 +1,5 @@ use ruff_diagnostics::{Diagnostic, Violation}; use ruff_macros::{derive_message_formats, violation}; -use ruff_python_index::Indexer; use ruff_python_trivia::CommentRanges; use ruff_source_file::Line; diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs index 606972bcf0c38..27320a43a3e1d 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs @@ -14,10 +14,9 @@ use std::fmt::{Debug, Formatter}; use std::iter::FusedIterator; use bitflags::bitflags; -use ruff_python_parser::lexer::LexResult; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; -use ruff_python_parser::TokenKind; +use ruff_python_parser::{TokenKind, Tokens}; use ruff_python_trivia::is_python_whitespace; use ruff_source_file::Locator; @@ -60,17 +59,16 @@ pub(crate) struct LogicalLines<'a> { } impl<'a> LogicalLines<'a> { - pub(crate) fn from_tokens(tokens: &'a [LexResult], locator: &'a Locator<'a>) -> Self { + pub(crate) fn from_tokens(tokens: &Tokens, locator: &'a Locator<'a>) -> Self { assert!(u32::try_from(tokens.len()).is_ok()); let mut builder = LogicalLinesBuilder::with_capacity(tokens.len()); let mut parens = 0u32; - for (token, range) in tokens.iter().flatten() { - let token_kind = TokenKind::from_token(token); - builder.push_token(token_kind, *range); + for token in tokens.up_to_first_unknown() { + builder.push_token(token.kind(), token.range()); - match token_kind { + match token.kind() { TokenKind::Lbrace | TokenKind::Lpar | TokenKind::Lsqb => { parens = parens.saturating_add(1); } @@ -506,9 +504,7 @@ struct Line { #[cfg(test)] mod tests { - use ruff_python_parser::lexer::LexResult; - use ruff_python_parser::{lexer, Mode}; - + use ruff_python_parser::parse_module; use ruff_source_file::Locator; use super::LogicalLines; @@ -592,9 +588,9 @@ if False: } fn assert_logical_lines(contents: &str, expected: &[&str]) { - let lxr: Vec = lexer::lex(contents, Mode::Module).collect(); + let program = parse_module(contents).unwrap(); let locator = Locator::new(contents); - let actual: Vec = LogicalLines::from_tokens(&lxr, &locator) + let actual: Vec = LogicalLines::from_tokens(program.tokens(), &locator) .into_iter() .map(|line| line.text_trimmed()) .map(ToString::to_string) diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/too_many_newlines_at_end_of_file.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/too_many_newlines_at_end_of_file.rs index 0117ff55186fa..c34ce2216bc5a 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/too_many_newlines_at_end_of_file.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/too_many_newlines_at_end_of_file.rs @@ -1,7 +1,7 @@ use ruff_diagnostics::{AlwaysFixableViolation, Diagnostic, Edit, Fix}; use ruff_macros::{derive_message_formats, violation}; use ruff_python_parser::{TokenKind, Tokens}; -use ruff_text_size::{TextRange, TextSize}; +use ruff_text_size::{Ranged, TextRange, TextSize}; /// ## What it does /// Checks for files with multiple trailing blank lines. diff --git a/crates/ruff_linter/src/rules/pyflakes/mod.rs b/crates/ruff_linter/src/rules/pyflakes/mod.rs index 90b12e5085015..b5c87f9351f21 100644 --- a/crates/ruff_linter/src/rules/pyflakes/mod.rs +++ b/crates/ruff_linter/src/rules/pyflakes/mod.rs @@ -17,12 +17,12 @@ mod tests { use ruff_python_ast::PySourceType; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; - use ruff_python_parser::AsMode; + use ruff_python_trivia::textwrap::dedent; use ruff_source_file::Locator; use ruff_text_size::Ranged; - use crate::linter::{check_path, LinterResult, TokenSource}; + use crate::linter::{check_path, LinterResult}; use crate::registry::{AsRule, Linter, Rule}; use crate::rules::pyflakes; use crate::settings::types::PreviewMode; @@ -663,7 +663,7 @@ mod tests { flags::Noqa::Enabled, &source_kind, source_type, - program, + &program, ); diagnostics.sort_by_key(Ranged::start); let actual = diagnostics diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs b/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs index 42214c6864149..65e1efd2e0dc6 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs @@ -7,7 +7,6 @@ use ruff_python_ast::parenthesize::parenthesized_range; use ruff_python_ast::{self as ast, Stmt}; use ruff_python_parser::{TokenKind, Tokens}; use ruff_python_semantic::{Binding, Scope}; -use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::checkers::ast::Checker; diff --git a/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs b/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs index 482d8a9744ce3..a923c55cd8f75 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs @@ -259,8 +259,8 @@ mod tests { use super::num_branches; fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> { - let branches = parse_module(source)?.suite(); - assert_eq!(num_branches(branches), expected_num_branches); + let program = parse_module(source)?; + assert_eq!(num_branches(program.suite()), expected_num_branches); Ok(()) } diff --git a/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs b/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs index 767fd0fa33b77..d03cdf735b309 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs @@ -103,8 +103,8 @@ mod tests { use super::num_returns; fn test_helper(source: &str, expected: usize) -> Result<()> { - let stmts = parse_module(source)?.suite(); - assert_eq!(num_returns(stmts), expected); + let program = parse_module(source)?; + assert_eq!(num_returns(program.suite()), expected); Ok(()) } diff --git a/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs b/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs index ce434da117403..3ab6f9fb15d1b 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs @@ -165,7 +165,7 @@ mod tests { use super::num_statements; fn parse_suite(source: &str) -> Result { - parse_module(source).map(|program| program.into_suite()) + Ok(parse_module(source)?.into_suite()) } #[test] diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/deprecated_import.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/deprecated_import.rs index 2c3ac4b9093bc..7e3e01f14aeea 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/deprecated_import.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/deprecated_import.rs @@ -1,5 +1,5 @@ use itertools::Itertools; -use ruff_python_ast::{Alias, Stmt, StmtImportFrom}; +use ruff_python_ast::{Alias, StmtImportFrom}; use ruff_diagnostics::{Diagnostic, Edit, Fix, FixAvailability, Violation}; use ruff_macros::{derive_message_formats, violation}; diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs index 2a93dcbf19c4d..bc75dbe6a7168 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs @@ -38,7 +38,7 @@ impl AlwaysFixableViolation for ExtraneousParentheses { } // See: https://github.com/asottile/pyupgrade/blob/97ed6fb3cf2e650d4f762ba231c3f04c41797710/pyupgrade/_main.py#L148 -fn match_extraneous_parentheses(mut tokens: Iter<'_, Token>) -> Option<(TextRange, TextRange)> { +fn match_extraneous_parentheses(tokens: &mut Iter<'_, Token>) -> Option<(TextRange, TextRange)> { // Store the location of the extraneous opening parenthesis. let start_range = loop { let token = tokens.next()?; @@ -70,10 +70,10 @@ fn match_extraneous_parentheses(mut tokens: Iter<'_, Token>) -> Option<(TextRang // If we find a comma or a yield at depth 1 or 2, it's a tuple or coroutine. TokenKind::Comma | TokenKind::Yield if depth == 1 => return None, TokenKind::Lpar | TokenKind::Lbrace | TokenKind::Lsqb => { - depth = depth.saturating_add(1) + depth = depth.saturating_add(1); } TokenKind::Rpar | TokenKind::Rbrace | TokenKind::Rsqb => { - depth = depth.saturating_sub(1) + depth = depth.saturating_sub(1); } _ => {} } diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs index a637e7f914aeb..6ecffdcb27897 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs @@ -11,7 +11,7 @@ use ruff_python_ast::{self as ast, Expr, Keyword}; use ruff_python_literal::format::{ FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, }; -use ruff_python_parser::{lexer, Mode, Tok, TokenKind}; +use ruff_python_parser::TokenKind; use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextRange}; diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/redundant_open_modes.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/redundant_open_modes.rs index 31d6c8b929e21..21ea4e4ff7cde 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/redundant_open_modes.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/redundant_open_modes.rs @@ -6,7 +6,6 @@ use ruff_diagnostics::{AlwaysFixableViolation, Diagnostic, Edit, Fix}; use ruff_macros::{derive_message_formats, violation}; use ruff_python_ast::{self as ast, Expr}; use ruff_python_parser::{TokenKind, Tokens}; -use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextSize}; use crate::checkers::ast::Checker; @@ -80,7 +79,6 @@ pub(crate) fn redundant_open_modes(checker: &mut Checker, call: &ast::ExprCall) call, &keyword.value, mode.replacement_value(), - checker.locator(), checker.program().tokens(), )); } @@ -95,7 +93,6 @@ pub(crate) fn redundant_open_modes(checker: &mut Checker, call: &ast::ExprCall) call, mode_param, mode.replacement_value(), - checker.locator(), checker.program().tokens(), )); } @@ -150,7 +147,6 @@ fn create_diagnostic( call: &ast::ExprCall, mode_param: &Expr, replacement_value: Option<&str>, - locator: &Locator, tokens: &Tokens, ) -> Diagnostic { let mut diagnostic = Diagnostic::new( @@ -166,16 +162,14 @@ fn create_diagnostic( mode_param.range(), ))); } else { - diagnostic.try_set_fix(|| { - create_remove_param_fix(locator, call, mode_param, tokens).map(Fix::safe_edit) - }); + diagnostic + .try_set_fix(|| create_remove_param_fix(call, mode_param, tokens).map(Fix::safe_edit)); } diagnostic } fn create_remove_param_fix( - locator: &Locator, call: &ast::ExprCall, mode_param: &Expr, tokens: &Tokens, diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs index f327f77ce70bb..f9ce477469a05 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs @@ -119,7 +119,7 @@ fn match_encoding_arg(arguments: &Arguments) -> Option { /// Return a [`Fix`] replacing the call to encode with a byte string. fn replace_with_bytes_literal(locator: &Locator, call: &ast::ExprCall, tokens: &Tokens) -> Fix { // Build up a replacement string by prefixing all string tokens with `b`. - let mut replacement = String::with_capacity(call.range() + 1); + let mut replacement = String::with_capacity(call.range().len().to_usize() + 1); let mut prev = call.start(); for token in tokens.tokens_in_range(call.range()) { match token.kind() { diff --git a/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs b/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs index 4001f2c0a6958..34e7ff32517bd 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs @@ -345,7 +345,7 @@ impl<'a> MultilineStringSequenceValue<'a> { // // Step (1). Start by collecting information on each line individually: let (lines, ends_with_trailing_comma) = - collect_string_sequence_lines(range, kind, locator, tokens, string_items)?; + collect_string_sequence_lines(range, kind, tokens, string_items)?; // Step (2). Group lines together into sortable "items": // - Any "item" contains a single element of the list/tuple @@ -489,7 +489,6 @@ impl Ranged for MultilineStringSequenceValue<'_> { fn collect_string_sequence_lines<'a>( range: TextRange, kind: SequenceKind, - locator: &Locator, tokens: &Tokens, string_items: &[&'a str], ) -> Option<(Vec>, bool)> { diff --git a/crates/ruff_linter/src/test.rs b/crates/ruff_linter/src/test.rs index 2b6d567cf2809..e68de264df5fe 100644 --- a/crates/ruff_linter/src/test.rs +++ b/crates/ruff_linter/src/test.rs @@ -16,14 +16,13 @@ use ruff_notebook::NotebookError; use ruff_python_ast::PySourceType; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::AsMode; use ruff_python_trivia::textwrap::dedent; use ruff_source_file::{Locator, SourceFileBuilder}; use ruff_text_size::Ranged; use crate::directives; use crate::fix::{fix_file, FixResult}; -use crate::linter::{check_path, LinterResult, TokenSource}; +use crate::linter::{check_path, LinterResult}; use crate::message::{Emitter, EmitterContext, Message, TextEmitter}; use crate::packaging::detect_package_root; use crate::registry::AsRule; @@ -136,7 +135,7 @@ pub(crate) fn test_contents<'a>( flags::Noqa::Enabled, source_kind, source_type, - program, + &program, ); let source_has_errors = error.is_some(); @@ -177,7 +176,7 @@ pub(crate) fn test_contents<'a>( transformed = Cow::Owned(transformed.updated(fixed_contents, &source_map)); let program = - ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); + ruff_python_parser::parse_unchecked_source(transformed.source_code(), source_type); let locator = Locator::new(transformed.source_code()); let stylist = Stylist::from_tokens(program.tokens(), &locator); let indexer = Indexer::from_tokens(program.tokens(), &locator); @@ -202,7 +201,7 @@ pub(crate) fn test_contents<'a>( flags::Noqa::Enabled, &transformed, source_type, - program, + &program, ); if let Some(fixed_error) = fixed_error { diff --git a/crates/ruff_python_codegen/Cargo.toml b/crates/ruff_python_codegen/Cargo.toml index 7afd304046f25..cf273027bb79d 100644 --- a/crates/ruff_python_codegen/Cargo.toml +++ b/crates/ruff_python_codegen/Cargo.toml @@ -18,6 +18,7 @@ ruff_python_ast = { workspace = true } ruff_python_literal = { workspace = true } ruff_python_parser = { workspace = true } ruff_source_file = { workspace = true } +ruff_text_size = { workspace = true } once_cell = { workspace = true } diff --git a/crates/ruff_python_codegen/src/stylist.rs b/crates/ruff_python_codegen/src/stylist.rs index aeb9ac20f3dcd..901a07fdbec86 100644 --- a/crates/ruff_python_codegen/src/stylist.rs +++ b/crates/ruff_python_codegen/src/stylist.rs @@ -98,7 +98,7 @@ fn detect_indention(tokens: &[Token], locator: &Locator) -> Indentation { // ) // ``` let mut depth = 0usize; - for (token, range) in tokens.iter().flatten() { + for token in tokens { match token.kind() { TokenKind::Lpar | TokenKind::Lbrace | TokenKind::Lsqb => { depth = depth.saturating_add(1); @@ -107,7 +107,7 @@ fn detect_indention(tokens: &[Token], locator: &Locator) -> Indentation { depth = depth.saturating_sub(1); } TokenKind::NonLogicalNewline => { - let line = locator.line(range.end()); + let line = locator.line(token.end()); let indent_index = line.chars().position(|c| !c.is_whitespace()); if let Some(indent_index) = indent_index { if indent_index > 0 { @@ -212,7 +212,7 @@ x = ( let locator = Locator::new(contents); let program = parse_module(contents).unwrap(); let stylist = Stylist::from_tokens(program.tokens(), &locator); - assert_eq!(stylist.indentation(), &Indentation::default()); + assert_eq!(stylist.indentation(), &Indentation(" ".to_string())); // formfeed indent, see `detect_indention` comment. let contents = r" diff --git a/crates/ruff_python_formatter/src/cli.rs b/crates/ruff_python_formatter/src/cli.rs index e7b1835824103..acffb5f4e2f5e 100644 --- a/crates/ruff_python_formatter/src/cli.rs +++ b/crates/ruff_python_formatter/src/cli.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; -use anyhow::{format_err, Context, Result}; +use anyhow::{Context, Result}; use clap::{command, Parser, ValueEnum}; use ruff_formatter::SourceCode; diff --git a/crates/ruff_python_formatter/src/comments/mod.rs b/crates/ruff_python_formatter/src/comments/mod.rs index ef775a1117967..0e9bfa63d1c7c 100644 --- a/crates/ruff_python_formatter/src/comments/mod.rs +++ b/crates/ruff_python_formatter/src/comments/mod.rs @@ -481,14 +481,12 @@ mod tests { use ruff_formatter::SourceCode; use ruff_python_ast::{Mod, PySourceType}; - use ruff_python_parser::{parse, AsMode}; - use ruff_python_trivia::CommentRanges; + use ruff_python_parser::{parse, AsMode, Program}; use crate::comments::Comments; struct CommentsTestCase<'a> { - module: &'a Mod, - comment_ranges: &'a CommentRanges, + program: Program, source_code: SourceCode<'a>, } @@ -500,14 +498,17 @@ mod tests { parse(source, source_type.as_mode()).expect("Expect source to be valid Python"); CommentsTestCase { + program, source_code, - module: program.syntax(), - comment_ranges: program.comment_ranges(), } } fn to_comments(&self) -> Comments { - Comments::from_ast(&self.module, self.source_code, &self.comment_ranges) + Comments::from_ast( + self.program.syntax(), + self.source_code, + self.program.comment_ranges(), + ) } } diff --git a/crates/ruff_python_formatter/src/lib.rs b/crates/ruff_python_formatter/src/lib.rs index 49927c0d7f369..e95d885b8706f 100644 --- a/crates/ruff_python_formatter/src/lib.rs +++ b/crates/ruff_python_formatter/src/lib.rs @@ -6,7 +6,7 @@ use ruff_formatter::prelude::*; use ruff_formatter::{format, write, FormatError, Formatted, PrintError, Printed, SourceCode}; use ruff_python_ast::AstNode; use ruff_python_ast::Mod; -use ruff_python_parser::{parse, AsMode, ParseError, ParseErrorType, Program}; +use ruff_python_parser::{parse, AsMode, ParseError, Program}; use ruff_python_trivia::CommentRanges; use ruff_source_file::Locator; diff --git a/crates/ruff_python_formatter/src/range.rs b/crates/ruff_python_formatter/src/range.rs index 75479ba26e464..7215e38ed5f67 100644 --- a/crates/ruff_python_formatter/src/range.rs +++ b/crates/ruff_python_formatter/src/range.rs @@ -5,9 +5,8 @@ use ruff_formatter::{ format, FormatContext, FormatError, FormatOptions, IndentStyle, PrintedRange, SourceCode, }; use ruff_python_ast::visitor::preorder::{walk_body, PreorderVisitor, TraversalSignal}; -use ruff_python_ast::{AnyNode, AnyNodeRef, Stmt, StmtMatch, StmtTry}; -use ruff_python_index::tokens_and_ranges; -use ruff_python_parser::{parse, parse_tokens, AsMode, ParseError, ParseErrorType}; +use ruff_python_ast::{AnyNodeRef, Stmt, StmtMatch, StmtTry}; +use ruff_python_parser::{parse, AsMode}; use ruff_python_trivia::{indentation_at_offset, BackwardsTokenizer, SimpleToken, SimpleTokenKind}; use ruff_source_file::Locator; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; diff --git a/crates/ruff_python_formatter/src/statement/suite.rs b/crates/ruff_python_formatter/src/statement/suite.rs index b28835caa540d..f8cb5e89a20b5 100644 --- a/crates/ruff_python_formatter/src/statement/suite.rs +++ b/crates/ruff_python_formatter/src/statement/suite.rs @@ -830,7 +830,6 @@ impl Format> for SuiteChildStatement<'_> { mod tests { use ruff_formatter::format; use ruff_python_parser::parse_module; - use ruff_python_trivia::CommentRanges; use crate::comments::Comments; use crate::prelude::*; diff --git a/crates/ruff_python_formatter/src/string/docstring.rs b/crates/ruff_python_formatter/src/string/docstring.rs index e0bb8d7e26d28..fa8e2d38fc061 100644 --- a/crates/ruff_python_formatter/src/string/docstring.rs +++ b/crates/ruff_python_formatter/src/string/docstring.rs @@ -9,7 +9,6 @@ use itertools::Itertools; use ruff_formatter::printer::SourceMapGeneration; use ruff_python_ast::{str::Quote, StringFlags}; -use ruff_python_parser::ParseError; use {once_cell::sync::Lazy, regex::Regex}; use { ruff_formatter::{write, FormatOptions, IndentStyle, LineWidth, Printed}, diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 08aa223403587..782820e56fdf2 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -7,7 +7,7 @@ use crate::TokenKind; /// Represents represent errors that occur during parsing and are /// returned by the `parse_*` functions. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct ParseError { pub error: ParseErrorType, pub location: TextRange, @@ -85,7 +85,7 @@ impl std::fmt::Display for FStringErrorType { } /// Represents the different types of errors that can occur during parsing. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub enum ParseErrorType { /// An unexpected error occurred. OtherError(String), diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index fdef12fa987c2..f85d5a8f30389 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -34,11 +34,6 @@ mod cursor; mod fstring; mod indentation; -#[deprecated] -pub fn lex(_source: &str, _mode: Mode) {} -#[deprecated] -pub fn lex_starts_at(_source: &str, _mode: Mode, _offset: TextSize) {} - /// A lexer for Python source code. #[derive(Debug)] pub struct Lexer<'src> { @@ -837,7 +832,7 @@ impl<'src> Lexer<'src> { } /// Lex the next token. - pub(crate) fn next_token(&mut self) -> TokenKind { + pub fn next_token(&mut self) -> TokenKind { self.cursor.start_token(); self.current_value = TokenValue::None; self.current_flags = TokenFlags::empty(); @@ -1874,6 +1869,11 @@ impl<'a> LexedText<'a> { } } +/// Create a new [`Lexer`] for the given source code and [`Mode`]. +pub fn lex(source: &str, mode: Mode) -> Lexer { + Lexer::new(source, mode, TextSize::default()) +} + #[cfg(test)] mod tests { use std::fmt::Write; diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index ce2dfc9e294fb..9613e8452d1dd 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -87,9 +87,6 @@ mod token_set; mod token_source; pub mod typing; -#[deprecated] -pub fn tokenize(_source: &str, _mode: Mode) {} - /// Parse a full Python module usually consisting of multiple lines. /// /// This is a convenience function that can be used to parse a full Python program without having to @@ -229,7 +226,7 @@ pub fn parse_unchecked(source: &str, mode: Mode) -> Program { Parser::new(source, mode).parse() } -/// Parse the given Python source code using the specificed [`PySourceType`]. +/// Parse the given Python source code using the specified [`PySourceType`]. pub fn parse_unchecked_source(source: &str, source_type: PySourceType) -> Program { // SAFETY: Safe because `PySourceType` always parses to a `ModModule` Parser::new(source, source_type.as_mode()) @@ -239,7 +236,7 @@ pub fn parse_unchecked_source(source: &str, source_type: PySourceType) -> Progra } /// Represents the parsed source code. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Program { syntax: T, tokens: Tokens, @@ -278,14 +275,29 @@ impl Program { self.errors } + /// Consumes the [`Program`] and returns the comment ranges found during parsing. + pub fn into_comment_ranges(self) -> CommentRanges { + self.comment_ranges + } + /// Returns `true` if the program is valid i.e., it has no syntax errors. pub fn is_valid(&self) -> bool { self.errors.is_empty() } - /// Transforms the [`Program`] into a [`Result`], returning [`Ok`] if the program has no syntax + /// Converts the [`Program`] into a [`Result`], returning [`Ok`] if the program has no syntax /// errors, or [`Err`] containing the first [`ParseError`] encountered. - pub fn into_result(self) -> Result, ParseError> { + pub fn as_result(&self) -> Result<&Program, &ParseError> { + if let [error, ..] = self.errors() { + Err(error) + } else { + Ok(self) + } + } + + /// Consumes the [`Program`] and returns a [`Result`] which is [`Ok`] if the program has no + /// syntax errors, or [`Err`] containing the first [`ParseError`] encountered. + pub(crate) fn into_result(self) -> Result, ParseError> { if self.is_valid() { Ok(self) } else { @@ -359,7 +371,7 @@ impl Program { } /// Tokens represents a vector of lexed [`Token`]. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Tokens { raw: Vec, @@ -492,6 +504,15 @@ impl Tokens { } } +impl<'a> IntoIterator for &'a Tokens { + type Item = &'a Token; + type IntoIter = std::slice::Iter<'a, Token>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + impl Deref for Tokens { type Target = [Token]; diff --git a/crates/ruff_python_parser/tests/fixtures.rs b/crates/ruff_python_parser/tests/fixtures.rs index 5d52f94493545..6b00bff4deead 100644 --- a/crates/ruff_python_parser/tests/fixtures.rs +++ b/crates/ruff_python_parser/tests/fixtures.rs @@ -126,12 +126,14 @@ fn test_invalid_syntax(input_path: &Path) { #[allow(clippy::print_stdout)] fn parser_quick_test() { let source = "\ -data[*x,] +def foo() + pass "; let program = parse_unchecked(source, Mode::Module); println!("AST:\n----\n{:#?}", program.syntax()); + println!("Tokens:\n-------\n{:#?}", program.tokens()); if !program.is_valid() { println!("Errors:\n-------"); diff --git a/crates/ruff_python_semantic/src/analyze/type_inference.rs b/crates/ruff_python_semantic/src/analyze/type_inference.rs index 33ce82a5aba00..4d15c0c36de90 100644 --- a/crates/ruff_python_semantic/src/analyze/type_inference.rs +++ b/crates/ruff_python_semantic/src/analyze/type_inference.rs @@ -428,7 +428,7 @@ impl NumberLike { #[cfg(test)] mod tests { - use ruff_python_ast::{Expr, ModExpression}; + use ruff_python_ast::ModExpression; use ruff_python_parser::{parse_expression, Program}; use crate::analyze::type_inference::{NumberLike, PythonType, ResolvedPythonType}; diff --git a/crates/ruff_python_trivia_integration_tests/tests/block_comments.rs b/crates/ruff_python_trivia_integration_tests/tests/block_comments.rs index 66ae74200f9a8..884c4362d09f1 100644 --- a/crates/ruff_python_trivia_integration_tests/tests/block_comments.rs +++ b/crates/ruff_python_trivia_integration_tests/tests/block_comments.rs @@ -1,5 +1,4 @@ -use ruff_python_index::Indexer; -use ruff_python_parser::{parse_module, tokenize, Mode}; +use ruff_python_parser::{parse_unchecked, Mode}; use ruff_source_file::Locator; use ruff_text_size::TextSize; @@ -7,7 +6,7 @@ use ruff_text_size::TextSize; fn block_comments_two_line_block_at_start() { // arrange let source = "# line 1\n# line 2\n"; - let program = parse_module(source).unwrap(); + let program = parse_unchecked(source, Mode::Module); let locator = Locator::new(source); // act @@ -21,7 +20,7 @@ fn block_comments_two_line_block_at_start() { fn block_comments_indented_block() { // arrange let source = " # line 1\n # line 2\n"; - let program = parse_module(source).unwrap(); + let program = parse_unchecked(source, Mode::Module); let locator = Locator::new(source); // act @@ -35,7 +34,7 @@ fn block_comments_indented_block() { fn block_comments_single_line_is_not_a_block() { // arrange let source = "\n"; - let program = parse_module(source).unwrap(); + let program = parse_unchecked(source, Mode::Module); let locator = Locator::new(source); // act @@ -49,7 +48,7 @@ fn block_comments_single_line_is_not_a_block() { fn block_comments_lines_with_code_not_a_block() { // arrange let source = "x = 1 # line 1\ny = 2 # line 2\n"; - let program = parse_module(source).unwrap(); + let program = parse_unchecked(source, Mode::Module); let locator = Locator::new(source); // act @@ -63,7 +62,7 @@ fn block_comments_lines_with_code_not_a_block() { fn block_comments_sequential_lines_not_in_block() { // arrange let source = " # line 1\n # line 2\n"; - let program = parse_module(source).unwrap(); + let program = parse_unchecked(source, Mode::Module); let locator = Locator::new(source); // act @@ -82,7 +81,7 @@ fn block_comments_lines_in_triple_quotes_not_a_block() { # line 2 """ "#; - let program = parse_module(source).unwrap(); + let program = parse_unchecked(source, Mode::Module); let locator = Locator::new(source); // act @@ -118,7 +117,7 @@ y = 2 # do not form a block comment # therefore do not form a block comment """ "#; - let program = parse_module(source).unwrap(); + let program = parse_unchecked(source, Mode::Module); let locator = Locator::new(source); // act diff --git a/crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs b/crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs index ceae7cc474c22..7010ec56316da 100644 --- a/crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs +++ b/crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs @@ -1,7 +1,6 @@ use insta::assert_debug_snapshot; -use ruff_python_parser::lexer::lex; -use ruff_python_parser::{parse_module, Mode, Tok}; +use ruff_python_parser::{parse_unchecked, Mode}; use ruff_python_trivia::{lines_after, lines_before, SimpleToken, SimpleTokenizer}; use ruff_python_trivia::{BackwardsTokenizer, SimpleTokenKind}; use ruff_text_size::{TextLen, TextRange, TextSize}; @@ -23,7 +22,7 @@ impl TokenizationTestCase { } fn tokenize_reverse(&self) -> Vec { - let program = parse_module(self.source).expect("Input to be a valid Python program"); + let program = parse_unchecked(self.source, Mode::Module); BackwardsTokenizer::new(self.source, self.range, program.comment_ranges()).collect() } diff --git a/crates/ruff_server/src/lint.rs b/crates/ruff_server/src/lint.rs index 0befdb0d966ce..74b6295f969b3 100644 --- a/crates/ruff_server/src/lint.rs +++ b/crates/ruff_server/src/lint.rs @@ -13,7 +13,6 @@ use ruff_linter::{ use ruff_notebook::Notebook; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::AsMode; use ruff_source_file::{LineIndex, Locator}; use ruff_text_size::{Ranged, TextRange}; use ruff_workspace::resolver::match_any_exclusion; @@ -120,7 +119,7 @@ pub(crate) fn check(query: &DocumentQuery, encoding: PositionEncoding) -> Diagno flags::Noqa::Enabled, &source_kind, source_type, - program, + &program, ); let noqa_edits = generate_noqa_edits( diff --git a/crates/ruff_wasm/src/lib.rs b/crates/ruff_wasm/src/lib.rs index cd3afd2696040..d09d59e93cbab 100644 --- a/crates/ruff_wasm/src/lib.rs +++ b/crates/ruff_wasm/src/lib.rs @@ -8,7 +8,7 @@ use ruff_formatter::printer::SourceMapGeneration; use ruff_formatter::{FormatResult, Formatted, IndentStyle}; use ruff_linter::directives; use ruff_linter::line_width::{IndentWidth, LineLength}; -use ruff_linter::linter::{check_path, LinterResult, TokenSource}; +use ruff_linter::linter::{check_path, LinterResult}; use ruff_linter::registry::AsRule; use ruff_linter::settings::types::PythonVersion; use ruff_linter::settings::{flags, DEFAULT_SELECTORS, DUMMY_VARIABLE_RGX}; @@ -17,8 +17,7 @@ use ruff_python_ast::{Mod, PySourceType}; use ruff_python_codegen::Stylist; use ruff_python_formatter::{format_module_ast, pretty_comments, PyFormatContext, QuoteStyle}; use ruff_python_index::Indexer; -use ruff_python_parser::{parse, parse_unchecked, parse_unchecked_source, AsMode, Mode, Program}; -use ruff_python_trivia::CommentRanges; +use ruff_python_parser::{parse, parse_unchecked, parse_unchecked_source, Mode, Program}; use ruff_source_file::{Locator, SourceLocation}; use ruff_text_size::Ranged; use ruff_workspace::configuration::Configuration; @@ -194,7 +193,7 @@ impl Workspace { flags::Noqa::Enabled, &source_kind, source_type, - program, + &program, ); let source_code = locator.to_source_code(); @@ -262,7 +261,7 @@ impl Workspace { } pub fn tokens(&self, contents: &str) -> Result { - let program = ruff_python_parser::parse_module(contents)?; + let program = parse_unchecked(contents, Mode::Module); Ok(format!("{:#?}", program.tokens())) }