Merge pull request #221 from Goncalerta/parser-refactor-pest

Pest Parser

liquid-compiler/Cargo.toml

@@ -15,8 +15,9 @@ travis-ci = { repository = "cobalt-org/liquid-rust" }
 appveyor  = { repository = "johannhof/liquid-rust" }
 
 [dependencies]
-regex = "1.0"
-lazy_static = "1.0"
+pest = "2.0"
+pest_derive = "2.0"
+
 # Exposed in API
 liquid-error = { version = "0.16", path = "../liquid-error" }
 liquid-value = { version = "0.17", path = "../liquid-value" }

liquid-compiler/src/block.rs

@@ -1,9 +1,9 @@
 use liquid_interpreter::Renderable;
 
 use super::error::Result;
-use super::Element;
 use super::LiquidOptions;
-use super::Token;
+use super::TagBlock;
+use super::TagTokenIter;
 
 /// A trait for creating custom custom block-size tags (`{% if something %}{% endif %}`).
 /// This is a simple type alias for a function.
@@ -17,8 +17,8 @@ pub trait ParseBlock: Send + Sync + ParseBlockClone {
     fn parse(
         &self,
         tag_name: &str,
-        arguments: &[Token],
-        tokens: &[Element],
+        arguments: TagTokenIter,
+        block: TagBlock,
         options: &LiquidOptions,
     ) -> Result<Box<Renderable>>;
 }
@@ -42,7 +42,7 @@ impl Clone for Box<ParseBlock> {
     }
 }
 
-pub type FnParseBlock = fn(&str, &[Token], &[Element], &LiquidOptions) -> Result<Box<Renderable>>;
+pub type FnParseBlock = fn(&str, TagTokenIter, TagBlock, &LiquidOptions) -> Result<Box<Renderable>>;
 
 #[derive(Clone)]
 struct FnBlockParser {
@@ -59,8 +59,8 @@ impl ParseBlock for FnBlockParser {
     fn parse(
         &self,
         tag_name: &str,
-        arguments: &[Token],
-        tokens: &[Element],
+        arguments: TagTokenIter,
+        tokens: TagBlock,
         options: &LiquidOptions,
     ) -> Result<Box<Renderable>> {
         (self.parser)(tag_name, arguments, tokens, options)
@@ -82,8 +82,8 @@ impl ParseBlock for BoxedBlockParser {
     fn parse(
         &self,
         tag_name: &str,
-        arguments: &[Token],
-        tokens: &[Element],
+        arguments: TagTokenIter,
+        tokens: TagBlock,
         options: &LiquidOptions,
     ) -> Result<Box<Renderable>> {
         match self.parser {

liquid-compiler/src/grammar.pest

@@ -0,0 +1,67 @@
+WHITESPACE = _{" "}
+LINE_OR_SPACE = _{ NEWLINE | WHITESPACE }
+NON_WHITESPACE_CONTROL_HYPHEN = _{ !"-}}" ~ !"-%}" ~ "-" }
+LiquidFile = ${ SOI ~ Element* ~ EOI }
+
+// Element-level parsing
+Element = _{ Expression | Tag | Raw } 
+
+TagStart = _{ (LINE_OR_SPACE* ~ "{%-") | "{%" }
+TagEnd =   _{ ("-%}" ~ LINE_OR_SPACE*) | "%}" }
+TagInner = !{Identifier ~ TagToken*}
+ExpressionStart = _{ (LINE_OR_SPACE* ~ "{{-") | "{{" }
+ExpressionEnd =   _{ ("-}}" ~ LINE_OR_SPACE*) | "}}" }
+ExpressionInner = !{FilterChain}
+
+Tag = { TagStart ~ WHITESPACE* ~ TagInner ~ WHITESPACE* ~ TagEnd }
+Expression = { ExpressionStart ~ WHITESPACE* ~ ExpressionInner ~ WHITESPACE* ~ ExpressionEnd }
+// Not allowing Tag/Expression Start/End might become a problem 
+// for {% raw %}, {% comment %} and other blocks that don't parse
+// the elements inside with liquid: unclosed delimiters won't be accepted
+Raw = @{ (!(TagStart | ExpressionStart | TagEnd | ExpressionEnd) ~ ANY)+ }              
+
+
+// Inner parsing        
+Identifier = @{ (ASCII_ALPHA | "_" | NON_WHITESPACE_CONTROL_HYPHEN) ~ (ASCII_ALPHANUMERIC | "_" | NON_WHITESPACE_CONTROL_HYPHEN)* }
+
+Variable = ${ Identifier 
+            ~ ( ("." ~ Identifier)
+              | ("[" ~ WHITESPACE* ~ Value ~ WHITESPACE* ~ "]")
+              )* 
+            }
+Value = { Literal | Variable }
+Filter = { Identifier ~ (":" ~ Value ~ ("," ~ Value)*)? } 
+FilterChain = { Value ~ ("|" ~ Filter)* }
+
+
+// Literals 
+StringLiteral = @{ ("'" ~ (!"'" ~ ANY)* ~ "'")
+		        		 | ("\"" ~ (!"\"" ~ ANY)* ~ "\"") }
+
+IntegerLiteral = @{ ("+" | "-")? ~ ASCII_DIGIT+ }
+FloatLiteral = @{ ("+" | "-")? ~ ASCII_DIGIT+ ~ "." ~ ASCII_DIGIT+ }
+
+BooleanLiteral = @{ "true" | "false" }
+
+Literal = { StringLiteral | FloatLiteral | IntegerLiteral | BooleanLiteral }
+
+Range = { "(" ~ Value ~ ".." ~ Value ~ ")" }
+
+TagToken = _{ Range | FilterChain | DoubleCharSymbol | SingleCharSymbol } 
+
+// DoubleCharSymbol must be tried first, otherwise it could be parsed as two SingleCharSymbol instead
+SingleCharSymbol = _{ GreaterThan | LesserThan | Assign | Comma | Colon }
+DoubleCharSymbol = _{ Equals | NotEquals | LesserThanGreaterThan | GreaterThanEquals | LesserThanEquals }
+
+// Symbols - Names must be given for better error messages
+GreaterThan = { ">" }
+LesserThan = { "<" }
+Assign = { "=" }
+Comma = { "," }
+Colon = { ":" }
+
+Equals = { "==" }
+NotEquals = { "!=" }
+LesserThanGreaterThan = { "<>" }
+GreaterThanEquals = { ">=" }
+LesserThanEquals = { "<=" }