Fix all compilation errors

crates/red_knot/src/parse.rs

@@ -32,17 +32,19 @@ impl Parsed {
         let result = ruff_python_parser::parse(text, Mode::Module);
 
         let (module, errors) = match result {
-            Ok(ast::Mod::Module(module)) => (module, vec![]),
-            Ok(ast::Mod::Expression(expression)) => (
-                ast::ModModule {
-                    range: expression.range(),
-                    body: vec![ast::Stmt::Expr(ast::StmtExpr {
+            Ok(parsed) => match parsed.into_syntax() {
+                ast::Mod::Module(module) => (module, vec![]),
+                ast::Mod::Expression(expression) => (
+                    ast::ModModule {
                         range: expression.range(),
-                        value: expression.body,
-                    })],
-                },
-                vec![],
-            ),
+                        body: vec![ast::Stmt::Expr(ast::StmtExpr {
+                            range: expression.range(),
+                            value: expression.body,
+                        })],
+                    },
+                    vec![],
+                ),
+            },
             Err(errors) => (
                 ast::ModModule {
                     range: TextRange::default(),

crates/ruff/src/diagnostics.rs

@@ -263,6 +263,7 @@ pub(crate) fn lint_path(
         LinterResult {
             data: messages,
             error: parse_error,
+            ..
         },
         transformed,
         fixed,
@@ -408,6 +409,7 @@ pub(crate) fn lint_stdin(
         LinterResult {
             data: messages,
             error: parse_error,
+            ..
         },
         transformed,
         fixed,

crates/ruff_benchmark/benches/lexer.rs

@@ -2,7 +2,7 @@ use ruff_benchmark::criterion::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
 };
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
-use ruff_python_parser::{lexer, Mode};
+use ruff_python_parser::{lexer, Mode, TokenKind};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -47,9 +47,15 @@ fn benchmark_lexer(criterion: &mut Criterion<WallTime>) {
             &case,
             |b, case| {
                 b.iter(|| {
-                    let result =
-                        lexer::lex(case.code(), Mode::Module).find(std::result::Result::is_err);
-                    assert_eq!(result, None, "Input to be a valid Python program");
+                    let mut lexer = lexer::lex(case.code(), Mode::Module);
+                    loop {
+                        let token = lexer.next_token();
+                        match token {
+                            TokenKind::EndOfFile => break,
+                            TokenKind::Unknown => panic!("Input to be a valid Python program"),
+                            _ => {}
+                        }
+                    }
                 });
             },
         );

crates/ruff_benchmark/benches/linter.rs

@@ -10,7 +10,7 @@ use ruff_linter::settings::{flags, LinterSettings};
 use ruff_linter::source_kind::SourceKind;
 use ruff_linter::{registry::Rule, RuleSelector};
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{parse_module, Mode};
+use ruff_python_parser::parse_module;
 
 #[cfg(target_os = "windows")]
 #[global_allocator]

crates/ruff_benchmark/benches/parser.rs

@@ -4,7 +4,7 @@ use ruff_benchmark::criterion::{
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_python_ast::Stmt;
-use ruff_python_parser::{parse_module, Mode};
+use ruff_python_parser::parse_module;
 
 #[cfg(target_os = "windows")]
 #[global_allocator]

crates/ruff_dev/Cargo.toml

@@ -22,6 +22,7 @@ ruff_python_formatter = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_python_stdlib = { workspace = true }
 ruff_python_trivia = { workspace = true }
+ruff_text_size = { workspace = true }
 ruff_workspace = { workspace = true, features = ["schemars"] }
 
 anyhow = { workspace = true }

crates/ruff_dev/src/print_tokens.rs

@@ -8,6 +8,7 @@ use anyhow::Result;
 use ruff_linter::source_kind::SourceKind;
 use ruff_python_ast::PySourceType;
 use ruff_python_parser::parse_unchecked_source;
+use ruff_text_size::Ranged;
 
 #[derive(clap::Args)]
 pub(crate) struct Args {
@@ -25,12 +26,12 @@ pub(crate) fn main(args: &Args) -> Result<()> {
         )
     })?;
     let program = parse_unchecked_source(source_kind.source_code(), source_type);
-    for token in program.tokens() {
+    for token in program.tokens().iter() {
         println!(
             "{start:#?} {kind:#?} {end:#?}",
             start = token.start(),
-            end = token.end()
-            kind = token.kind()
+            end = token.end(),
+            kind = token.kind(),
         );
     }
     Ok(())

crates/ruff_linter/src/checkers/imports.rs

@@ -4,10 +4,10 @@ use std::path::Path;
 use ruff_diagnostics::Diagnostic;
 use ruff_notebook::CellOffsets;
 use ruff_python_ast::statement_visitor::StatementVisitor;
-use ruff_python_ast::{ModModule, PySourceType, Suite};
+use ruff_python_ast::{ModModule, PySourceType};
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::{Program, Tokens};
+use ruff_python_parser::Program;
 use ruff_source_file::Locator;
 
 use crate::directives::IsortDirectives;

crates/ruff_linter/src/checkers/logical_lines.rs

@@ -2,8 +2,7 @@ use crate::line_width::IndentWidth;
 use ruff_diagnostics::Diagnostic;
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::lexer::LexResult;
-use ruff_python_parser::TokenKind;
+use ruff_python_parser::{TokenKind, Tokens};
 use ruff_source_file::Locator;
 use ruff_text_size::{Ranged, TextRange};
 
@@ -34,7 +33,7 @@ pub(crate) fn expand_indent(line: &str, indent_width: IndentWidth) -> usize {
 }
 
 pub(crate) fn check_logical_lines(
-    tokens: &[LexResult],
+    tokens: &Tokens,
     locator: &Locator,
     indexer: &Indexer,
     stylist: &Stylist,

crates/ruff_linter/src/checkers/tokens.rs

@@ -8,7 +8,7 @@ use ruff_python_codegen::Stylist;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_index::Indexer;
-use ruff_python_parser::{Program, Tokens};
+use ruff_python_parser::Program;
 use ruff_source_file::Locator;
 use ruff_text_size::Ranged;
 

crates/ruff_linter/src/directives.rs

@@ -4,9 +4,8 @@ use std::iter::Peekable;
 use std::str::FromStr;
 
 use bitflags::bitflags;
-use ruff_python_ast::{ModModule, StringFlags};
-use ruff_python_parser::lexer::LexResult;
-use ruff_python_parser::{Program, Tok, TokenKind, Tokens};
+use ruff_python_ast::ModModule;
+use ruff_python_parser::{Program, TokenKind, Tokens};
 use ruff_python_trivia::CommentRanges;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
@@ -60,7 +59,7 @@ pub fn extract_directives(
 ) -> Directives {
     Directives {
         noqa_line_for: if flags.intersects(Flags::NOQA) {
-            extract_noqa_line_for(lxr, locator, indexer)
+            extract_noqa_line_for(program.tokens(), locator, indexer)
         } else {
             NoqaMapping::default()
         },
@@ -380,8 +379,7 @@ impl TodoDirectiveKind {
 
 #[cfg(test)]
 mod tests {
-    use ruff_python_parser::lexer::LexResult;
-    use ruff_python_parser::{lexer, parse_module, Mode};
+    use ruff_python_parser::parse_module;
     use ruff_text_size::{TextLen, TextRange, TextSize};
 
     use ruff_python_index::Indexer;

crates/ruff_linter/src/fix/edits.rs

@@ -533,7 +533,7 @@ mod tests {
     use ruff_diagnostics::{Diagnostic, Edit, Fix};
     use ruff_python_ast::Stmt;
     use ruff_python_codegen::Stylist;
-    use ruff_python_parser::{lexer, parse, parse_expression, parse_module, Mode};
+    use ruff_python_parser::{parse_expression, parse_module};
     use ruff_source_file::Locator;
     use ruff_text_size::{Ranged, TextRange, TextSize};
 
@@ -615,7 +615,7 @@ x = 1 \
     }
 
     #[test]
-    fn redundant_alias() {
+    fn redundant_alias() -> Result<()> {
         let contents = "import x, y as y, z as bees";
         let stmt = parse_first_stmt(contents)?;
         assert_eq!(
@@ -642,6 +642,7 @@ x = 1 \
             )],
             "the third item is already aliased to something else"
         );
+        Ok(())
     }
 
     #[test_case("()",             &["x", "y"], r#"("x", "y")"#             ; "2 into empty tuple")]

crates/ruff_linter/src/importer/insertion.rs

@@ -1,8 +1,8 @@
 //! Insert statements into Python code.
 use std::ops::Add;
 
-use ruff_python_ast::{PySourceType, Stmt};
-use ruff_python_parser::{lexer, AsMode, Tok, TokenKind, Tokens};
+use ruff_python_ast::Stmt;
+use ruff_python_parser::{TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextSize};
 
 use ruff_diagnostics::Edit;
@@ -172,7 +172,7 @@ impl<'a> Insertion<'a> {
                 // Once we've seen the colon, we're looking for a newline; otherwise, there's no
                 // block body (e.g. `if True: pass`).
                 Awaiting::Newline => match token.kind() {
-                    TokenKind::Comment(..) => {}
+                    TokenKind::Comment => {}
                     TokenKind::Newline => {
                         state = Awaiting::Indent;
                     }
@@ -183,7 +183,7 @@ impl<'a> Insertion<'a> {
                 },
                 // Once we've seen the newline, we're looking for the indentation of the block body.
                 Awaiting::Indent => match token.kind() {
-                    TokenKind::Comment(..) => {}
+                    TokenKind::Comment => {}
                     TokenKind::NonLogicalNewline => {}
                     TokenKind::Indent => {
                         // This is like:
@@ -317,9 +317,8 @@ fn match_continuation(s: &str) -> Option<TextSize> {
 mod tests {
     use anyhow::Result;
 
-    use ruff_python_ast::PySourceType;
     use ruff_python_codegen::Stylist;
-    use ruff_python_parser::{parse, parse_module, Mode};
+    use ruff_python_parser::parse_module;
     use ruff_source_file::{LineEnding, Locator};
     use ruff_text_size::TextSize;