refactor: Remove common module (PRQL#4728)

prqlc/prqlc-parser/src/lexer/mod.rs

@@ -202,7 +202,7 @@ fn comment() -> impl Parser<char, TokenKind, Error = Cheap<char>> {
     )))
 }
 
-pub fn ident_part() -> impl Parser<char, String, Error = Cheap<char>> + Clone {
+pub(crate) fn ident_part() -> impl Parser<char, String, Error = Cheap<char>> + Clone {
     let plain = filter(|c: &char| c.is_alphabetic() || *c == '_')
         .chain(filter(|c: &char| c.is_alphanumeric() || *c == '_').repeated());
 

prqlc/prqlc-parser/src/parser/expr.rs

@@ -4,22 +4,20 @@ use chumsky::prelude::*;
 use itertools::Itertools;
 
 use crate::lexer::lr::{Literal, TokenKind};
-use crate::parser::common::{ctrl, ident_part, keyword, new_line, with_doc_comment};
 use crate::parser::interpolation;
 use crate::parser::perror::PError;
 use crate::parser::pr::*;
 use crate::parser::types::type_expr;
+use crate::parser::{ctrl, ident_part, keyword, new_line, sequence, with_doc_comment};
 use crate::span::Span;
 
-use super::common::sequence;
-
-pub fn expr_call() -> impl Parser<TokenKind, Expr, Error = PError> + Clone {
+pub(crate) fn expr_call() -> impl Parser<TokenKind, Expr, Error = PError> + Clone {
     let expr = expr();
 
     lambda_func(expr.clone()).or(func_call(expr))
 }
 
-pub fn expr() -> impl Parser<TokenKind, Expr, Error = PError> + Clone {
+pub(crate) fn expr() -> impl Parser<TokenKind, Expr, Error = PError> + Clone {
     recursive(|expr| {
         let literal = select! { TokenKind::Literal(lit) => ExprKind::Literal(lit) };
 
@@ -323,7 +321,7 @@ where
         .boxed()
 }
 
-pub fn binary_op_parser_right<'a, Term, Op>(
+pub(crate) fn binary_op_parser_right<'a, Term, Op>(
     term: Term,
     op: Op,
 ) -> impl Parser<TokenKind, Expr, Error = PError> + Clone + 'a

prqlc/prqlc-parser/src/parser/interpolation.rs

@@ -7,7 +7,7 @@ use crate::parser::pr::*;
 use crate::span::{string_stream, Span};
 
 /// Parses interpolated strings
-pub fn parse(string: String, span_base: Span) -> Result<Vec<InterpolateItem>, Vec<PError>> {
+pub(crate) fn parse(string: String, span_base: Span) -> Result<Vec<InterpolateItem>, Vec<PError>> {
     let prepped_stream = string_stream(string, span_base);
 
     let res = interpolated_parser().parse(prepped_stream);
@@ -66,7 +66,8 @@ fn interpolated_parser() -> impl Parser<char, Vec<InterpolateItem>, Error = Chum
     expr.or(string).repeated().then_ignore(end())
 }
 
-pub fn interpolate_ident_part() -> impl Parser<char, String, Error = ChumError<char>> + Clone {
+pub(crate) fn interpolate_ident_part() -> impl Parser<char, String, Error = ChumError<char>> + Clone
+{
     let plain = filter(|c: &char| c.is_alphabetic() || *c == '_')
         .chain(filter(|c: &char| c.is_alphanumeric() || *c == '_').repeated())
         .labelled("interpolated string");

prqlc/prqlc-parser/src/parser/mod.rs

@@ -1,11 +1,12 @@
 use chumsky::{prelude::*, Stream};
 
-pub use self::common::new_line;
+use self::perror::PError;
+use self::pr::{Annotation, Stmt, StmtKind};
 use crate::error::Error;
 use crate::lexer::lr;
+use crate::lexer::lr::TokenKind;
 use crate::span::Span;
 
-mod common;
 mod expr;
 mod interpolation;
 pub(crate) mod perror;
@@ -60,6 +61,140 @@ pub(crate) fn prepare_stream(
     Stream::from_iter(eoi, tokens)
 }
 
-pub trait SupportsDocComment {
+pub(crate) fn ident_part() -> impl Parser<TokenKind, String, Error = PError> + Clone {
+    select! {
+        TokenKind::Ident(ident) => ident,
+        TokenKind::Keyword(ident) if &ident == "module" => ident,
+    }
+    .map_err(|e: PError| {
+        PError::expected_input_found(
+            e.span(),
+            [Some(TokenKind::Ident("".to_string()))],
+            e.found().cloned(),
+        )
+    })
+}
+
+pub(crate) fn keyword(kw: &'static str) -> impl Parser<TokenKind, (), Error = PError> + Clone {
+    just(TokenKind::Keyword(kw.to_string())).ignored()
+}
+
+/// Our approach to new lines is each item consumes new lines _before_ itself,
+/// but not newlines after itself. This allows us to enforce new lines between
+/// some items. The only place we handle new lines after an item is in the root
+/// parser.
+pub(crate) fn new_line() -> impl Parser<TokenKind, (), Error = PError> + Clone {
+    just(TokenKind::NewLine)
+        // Start is considered a new line, so we can enforce things start on a new
+        // line while allowing them to be at the beginning of a file
+        .or(just(TokenKind::Start))
+        .ignored()
+        .labelled("new line")
+}
+
+pub(crate) fn ctrl(char: char) -> impl Parser<TokenKind, (), Error = PError> + Clone {
+    just(TokenKind::Control(char)).ignored()
+}
+
+pub(crate) fn into_stmt((annotations, kind): (Vec<Annotation>, StmtKind), span: Span) -> Stmt {
+    Stmt {
+        kind,
+        span: Some(span),
+        annotations,
+        doc_comment: None,
+    }
+}
+
+pub(crate) fn doc_comment() -> impl Parser<TokenKind, String, Error = PError> + Clone {
+    // doc comments must start on a new line, so we enforce a new line (which
+    // can also be a file start) before the doc comment
+    //
+    // TODO: we currently lose any empty newlines between doc comments;
+    // eventually we want to retain them
+    (new_line().repeated().at_least(1).ignore_then(select! {
+        TokenKind::DocComment(dc) => dc,
+    }))
+    .repeated()
+    .at_least(1)
+    .collect()
+    .map(|lines: Vec<String>| lines.join("\n"))
+    .labelled("doc comment")
+}
+
+pub(crate) fn with_doc_comment<'a, P, O>(
+    parser: P,
+) -> impl Parser<TokenKind, O, Error = PError> + Clone + 'a
+where
+    P: Parser<TokenKind, O, Error = PError> + Clone + 'a,
+    O: SupportsDocComment + 'a,
+{
+    doc_comment()
+        .or_not()
+        .then(parser)
+        .map(|(doc_comment, inner)| inner.with_doc_comment(doc_comment))
+}
+
+/// Allows us to surround a parser by `with_doc_comment` and for a doc comment
+/// to be added to the result, as long as the result implements `SupportsDocComment`.
+///
+/// We could manage without it tbh,
+pub(crate) trait SupportsDocComment {
     fn with_doc_comment(self, doc_comment: Option<String>) -> Self;
 }
+
+/// Parse a sequence, allowing commas and new lines between items. Doesn't
+/// include the surrounding delimiters.
+pub(crate) fn sequence<'a, P, O>(
+    parser: P,
+) -> impl Parser<TokenKind, Vec<O>, Error = PError> + Clone + 'a
+where
+    P: Parser<TokenKind, O, Error = PError> + Clone + 'a,
+    O: 'a,
+{
+    parser
+        .separated_by(ctrl(',').then_ignore(new_line().repeated()))
+        .allow_trailing()
+        .padded_by(new_line().repeated())
+}
+
+#[cfg(test)]
+mod tests {
+    use insta::assert_debug_snapshot;
+
+    use super::*;
+    use crate::test::parse_with_parser;
+
+    #[test]
+    fn test_doc_comment() {
+        assert_debug_snapshot!(parse_with_parser(r#"
+        #! doc comment
+        #! another line
+
+        "#, doc_comment()), @r###"
+        Ok(
+            " doc comment\n another line",
+        )
+        "###);
+    }
+
+    #[test]
+    fn test_doc_comment_or_not() {
+        assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not()).unwrap(), @"None");
+        assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not().then_ignore(new_line().repeated()).then(ident_part())).unwrap(), @r###"
+        (
+            None,
+            "hello",
+        )
+        "###);
+    }
+
+    #[test]
+    fn test_no_doc_comment_in_with_doc_comment() {
+        impl SupportsDocComment for String {
+            fn with_doc_comment(self, _doc_comment: Option<String>) -> Self {
+                self
+            }
+        }
+        assert_debug_snapshot!(parse_with_parser(r#"hello"#, with_doc_comment(new_line().ignore_then(ident_part()))).unwrap(), @r###""hello""###);
+    }
+}

prqlc/prqlc-parser/src/parser/pr/expr.rs

@@ -4,12 +4,11 @@ use enum_as_inner::EnumAsInner;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::generic;
 use crate::lexer::lr::Literal;
 use crate::parser::pr::ops::{BinOp, UnOp};
 use crate::parser::pr::Ty;
-use crate::parser::SupportsDocComment;
 use crate::span::Span;
+use crate::{generic, parser::SupportsDocComment};
 
 impl Expr {
     pub fn new<K: Into<ExprKind>>(kind: K) -> Self {

prqlc/prqlc-parser/src/parser/pr/mod.rs

@@ -10,9 +10,7 @@ pub use ops::*;
 pub use stmt::*;
 pub use types::*;
 
-// re-export Literal from LR, since it's encapsulated in TyKind
-//TODO: maybe remove these?
 pub use crate::generic;
+// re-export Literal from LR, since it's encapsulated in TyKind
 pub use crate::lexer::lr::Literal;
-pub use crate::lexer::lr::ValueAndUnit;
 pub use crate::span::Span;

prqlc/prqlc-parser/src/parser/pr/stmt.rs

@@ -5,8 +5,9 @@ use schemars::JsonSchema;
 use semver::VersionReq;
 use serde::{Deserialize, Serialize};
 
+use crate::parser::pr::ident::Ident;
 use crate::parser::pr::{Expr, Ty};
-use crate::parser::{pr::ident::Ident, SupportsDocComment};
+use crate::parser::SupportsDocComment;
 use crate::span::Span;
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default, JsonSchema)]