From 7e6e100786cfab35c8518d71e85e9e57ab5c6bda Mon Sep 17 00:00:00 2001
From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com>
Date: Sat, 27 Jul 2024 21:16:06 -0700
Subject: [PATCH] refactor: Remove `source` arg from parser (#4796)

---
 prqlc/prqlc-parser/src/parser/mod.rs  | 24 ++++++++++--------------
 prqlc/prqlc-parser/src/parser/stmt.rs |  2 +-
 prqlc/prqlc-parser/src/parser/test.rs |  2 +-
 prqlc/prqlc-parser/src/test.rs        |  2 +-
 prqlc/prqlc/src/parser.rs             |  2 +-
 5 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs
index 2c2f456fe8d8..356fdb4a8491 100644
--- a/prqlc/prqlc-parser/src/parser/mod.rs
+++ b/prqlc/prqlc-parser/src/parser/mod.rs
@@ -19,12 +19,8 @@ mod types;
 // Note that `parse_source` is in `prqlc` crate, not in `prqlc-parser` crate,
 // because it logs using the logging framework in `prqlc`.
 
-pub fn parse_lr_to_pr(
-    source: &str,
-    source_id: u16,
-    lr: Vec<lr::Token>,
-) -> (Option<Vec<pr::Stmt>>, Vec<Error>) {
-    let stream = prepare_stream(lr.into_iter(), source, source_id);
+pub fn parse_lr_to_pr(source_id: u16, lr: Vec<lr::Token>) -> (Option<Vec<pr::Stmt>>, Vec<Error>) {
+    let stream = prepare_stream(lr, source_id);
     let (pr, parse_errors) = stmt::source().parse_recovery(stream);
 
     let errors = parse_errors.into_iter().map(|e| e.into()).collect();
@@ -35,14 +31,15 @@ pub fn parse_lr_to_pr(
 
 /// Convert the output of the lexer into the input of the parser. Requires
 /// supplying the original source code.
-pub(crate) fn prepare_stream(
-    tokens: impl Iterator<Item = lr::Token>,
-    source: &str,
+pub(crate) fn prepare_stream<'a>(
+    tokens: Vec<lr::Token>,
     source_id: u16,
-) -> Stream<lr::TokenKind, Span, impl Iterator<Item = (lr::TokenKind, Span)> + Sized> {
+) -> Stream<'a, lr::TokenKind, Span, impl Iterator<Item = (lr::TokenKind, Span)> + Sized + 'a> {
+    let final_span = tokens.last().map(|t| t.span.end).unwrap_or(0);
+
     // We don't want comments in the AST (but we do intend to use them as part of
     // formatting)
-    let semantic_tokens = tokens.filter(|token| {
+    let semantic_tokens = tokens.into_iter().filter(|token| {
         !matches!(
             token.kind,
             lr::TokenKind::Comment(_) | lr::TokenKind::LineWrap(_)
@@ -52,10 +49,9 @@ pub(crate) fn prepare_stream(
     let tokens = semantic_tokens
         .into_iter()
         .map(move |token| (token.kind, Span::new(source_id, token.span)));
-    let len = source.chars().count();
     let eoi = Span {
-        start: len,
-        end: len + 1,
+        start: final_span,
+        end: final_span + 1,
         source_id,
     };
     Stream::from_iter(eoi, tokens)
diff --git a/prqlc/prqlc-parser/src/parser/stmt.rs b/prqlc/prqlc-parser/src/parser/stmt.rs
index 17b2e9654be7..a67a7cd9b6d8 100644
--- a/prqlc/prqlc-parser/src/parser/stmt.rs
+++ b/prqlc/prqlc-parser/src/parser/stmt.rs
@@ -307,7 +307,7 @@ mod tests {
             Error {
                 kind: Error,
                 span: Some(
-                    0:80-81,
+                    0:72-73,
                 ),
                 reason: Simple(
                     "unexpected end of input",
diff --git a/prqlc/prqlc-parser/src/parser/test.rs b/prqlc/prqlc-parser/src/parser/test.rs
index 8f39db6df197..42bd418f914f 100644
--- a/prqlc/prqlc-parser/src/parser/test.rs
+++ b/prqlc/prqlc-parser/src/parser/test.rs
@@ -27,7 +27,7 @@ fn test_prepare_stream() {
     let input = "from artists | filter name == 'John'";
     let tokens = lex_source(input).unwrap();
 
-    let mut stream = prepare_stream(tokens.0.into_iter(), input, 0);
+    let mut stream = prepare_stream(tokens.0, 0);
     assert_yaml_snapshot!(stream.fetch_tokens().collect::<Vec<(TokenKind, Span)>>(), @r###"
     ---
     - - Start
diff --git a/prqlc/prqlc-parser/src/test.rs b/prqlc/prqlc-parser/src/test.rs
index 620dedfdfbc7..10672c6837dc 100644
--- a/prqlc/prqlc-parser/src/test.rs
+++ b/prqlc/prqlc-parser/src/test.rs
@@ -15,7 +15,7 @@ pub(crate) fn parse_with_parser<O: Debug>(
     parser: impl Parser<TokenKind, O, Error = PError>,
 ) -> Result<O, Vec<Error>> {
     let tokens = crate::lexer::lex_source(source)?;
-    let stream = prepare_stream(tokens.0.into_iter(), source, 0);
+    let stream = prepare_stream(tokens.0, 0);
 
     // TODO: possibly should check we consume all the input? Either with an
     // end() parser or some other way (but if we add an end parser then this
diff --git a/prqlc/prqlc/src/parser.rs b/prqlc/prqlc/src/parser.rs
index 692574c97e82..4f612b7348ad 100644
--- a/prqlc/prqlc/src/parser.rs
+++ b/prqlc/prqlc/src/parser.rs
@@ -63,7 +63,7 @@ pub(crate) fn parse_source(source: &str, source_id: u16) -> Result<Vec<pr::Stmt>
     let ast = if let Some(tokens) = tokens {
         debug::log_entry(|| debug::DebugEntryKind::ReprLr(lr::Tokens(tokens.clone())));
 
-        let (ast, parse_errors) = prqlc_parser::parser::parse_lr_to_pr(source, source_id, tokens);
+        let (ast, parse_errors) = prqlc_parser::parser::parse_lr_to_pr(source_id, tokens);
         errors.extend(parse_errors);
         ast
     } else {