diff --git a/src/benchCurrent/Program.cs b/src/benchCurrent/Program.cs index 9290f8e3..0124c41f 100644 --- a/src/benchCurrent/Program.cs +++ b/src/benchCurrent/Program.cs @@ -8,11 +8,13 @@ static class Program private static void BenchJson() { - var summary = BenchmarkRunner.Run(); + // var summary = BenchmarkRunner.Run(); + // + // var summary2 = BenchmarkRunner.Run(); - var summary2 = BenchmarkRunner.Run(); - - var summary3 = BenchmarkRunner.Run(); + // var summary3 = BenchmarkRunner.Run(); + + var summary4 = BenchmarkRunner.Run(); } static void Main(string[] args) diff --git a/src/benchCurrent/SimpleExpressionBench.cs b/src/benchCurrent/SimpleExpressionBench.cs new file mode 100644 index 00000000..ac5e78df --- /dev/null +++ b/src/benchCurrent/SimpleExpressionBench.cs @@ -0,0 +1,114 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.CsProj; +using expressionparser; +using simpleExpressionParser; +using sly.parser; +using sly.parser.generator; +using ExpressionToken = simpleExpressionParser.ExpressionToken; + +namespace benchCurrent +{ + + [MemoryDiagnoser] + + [Config(typeof(Config))] + public class SimpleExpressionBench + { + + + private class Config : ManualConfig + { + public Config() + { + var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp70); + } + } + + private Parser BenchedParser; + private Parser BenchedGenericParser; + + private string content = ""; + + [GlobalSetup] + public void Setup() + { + Console.WriteLine(("SETUP")); + content = "1+2+3+4+5+6+7+8+9+10+11+12+13+14+15+16+17+18+19+20"; + + SimpleExpressionParser p = new SimpleExpressionParser(); + var builder = new ParserBuilder(); + + var result = builder.BuildParser(p, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + + if (result.IsError) + { + result.Errors.ForEach(x => Console.WriteLine(x.Message)); + Environment.Exit(1); + } + else + { + Console.WriteLine("parser ok"); + BenchedParser = result.Result; + } + + GenericSimpleExpressionParser gp = new GenericSimpleExpressionParser(); + var genericbuilder = new ParserBuilder(); + + var genericresult = genericbuilder.BuildParser(gp, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + + if (genericresult.IsError) + { + genericresult.Errors.ForEach(x => Console.WriteLine(x.Message)); + Environment.Exit(2); + } + else + { + Console.WriteLine("parser ok"); + BenchedGenericParser = genericresult.Result; + } + } + + + + [Benchmark] + + public void TestExpressionRegex() + { + if (BenchedParser == null) + { + Console.WriteLine("regex parser is null"); + } + else + { + for (int i = 0; i < 50; i++) + { + var ignored = BenchedParser.Parse(content); + } + } + } + + // [Benchmark] + + public void TestExpressionGeneric() + { + if (BenchedGenericParser == null) + { + Console.WriteLine("generic parser is null"); + } + else + { + for (int i = 0; i < 50; i++) + { + var ignored = BenchedGenericParser.Parse(content); + } + } + } + + + + } + +} diff --git a/src/benchCurrent/benchCurrent.csproj b/src/benchCurrent/benchCurrent.csproj index d2834772..5da9d7e0 100644 --- a/src/benchCurrent/benchCurrent.csproj +++ b/src/benchCurrent/benchCurrent.csproj @@ -27,7 +27,9 @@ + + diff --git a/src/match.dmw b/src/match.dmw new file mode 100644 index 00000000..c5295b1c Binary files /dev/null and b/src/match.dmw differ diff --git a/src/samples/ParserExample/Program.cs b/src/samples/ParserExample/Program.cs index 4446011c..861def36 100644 --- a/src/samples/ParserExample/Program.cs +++ b/src/samples/ParserExample/Program.cs @@ -38,6 +38,7 @@ using XML; using Xunit; using ExpressionContext = postProcessedLexerParser.expressionModel.ExpressionContext; +using ExpressionToken = simpleExpressionParser.ExpressionToken; using IfThenElse = indented.IfThenElse; namespace ParserExample @@ -131,6 +132,26 @@ public static object Rec(List args) } + private static void BenchSimpleExpression() + { + GenericSimpleExpressionParser p = new GenericSimpleExpressionParser(); + var builder = new ParserBuilder(); + + var Parser = builder.BuildParser(p, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + if (Parser.IsOk) + { + for (int i = 0; i < 50; i++) + { + var r = Parser.Result.Parse("1+2+3+4+5+6+7+8+9+10+11+12+13+14+15+16+17+18+19+20"); + if (r.IsOk) + { + Console.WriteLine(r.Result); + } + } + + } + } + private static void TestFactorial() { var whileParser = new WhileParserGeneric(); @@ -1183,7 +1204,8 @@ print a } private static void Main(string[] args) { - IndentRefactoring(); + BenchSimpleExpression(); + // IndentRefactoring(); //NodeNames(); // BroadWindow(); // return; diff --git a/src/samples/SimpleExpressionParser/ExpressionToken.cs b/src/samples/SimpleExpressionParser/ExpressionToken.cs new file mode 100644 index 00000000..f223b783 --- /dev/null +++ b/src/samples/SimpleExpressionParser/ExpressionToken.cs @@ -0,0 +1,58 @@ +using sly.i18n; +using sly.lexer; + +namespace simpleExpressionParser +{ + public enum ExpressionToken + { + // float number + [Lexeme("[0-9]+\\.[0-9]+")] DOUBLE = 1, + + // integer + [Lexeme("[0-9]+")] INT = 3, + + [Lexeme("[a-zA-Z]+")] IDENTIFIER = 4, + + // the + operator + [LexemeLabel("en","plus sign")] + [LexemeLabel("fr","plus")] + [Lexeme("\\+")] PLUS = 5, + + // the - operator + [LexemeLabel("en","minus sign")] + [LexemeLabel("fr","moins")] + [Lexeme("\\-")] MINUS = 6, + + // the * operator + [LexemeLabel("en","times sign")] + [LexemeLabel("fr","multiplication")] + [Lexeme("\\*")] TIMES = 7, + + // the / operator + [LexemeLabel("en","divide sign")] + [LexemeLabel("fr","division")] + [Lexeme("\\/")] DIVIDE = 8, + + // a left paranthesis ( + [LexemeLabel("en","opening parenthesis")] + [LexemeLabel("fr","parenthèse ouvrante")] + [Lexeme("\\(")] LPAREN = 9, + + // a right paranthesis ) + [LexemeLabel("en","closing parenthesis")] + [LexemeLabel("fr","parenthèse fermante")] + [Lexeme("\\)")] RPAREN = 10, + + [LexemeLabel("fr","point d'exclamation")] + [LexemeLabel("en","exclamation point")] + [Lexeme("!")] FACTORIAL = 13, + + + // a whitespace + [Lexeme("[ \\t]+", true)] WS = 11, + + [Lexeme("[\\n\\r]+", true, true)] EOL = 12 + + + } +} diff --git a/src/samples/SimpleExpressionParser/GenericExpressionToken.cs b/src/samples/SimpleExpressionParser/GenericExpressionToken.cs new file mode 100644 index 00000000..869b2cef --- /dev/null +++ b/src/samples/SimpleExpressionParser/GenericExpressionToken.cs @@ -0,0 +1,44 @@ +using sly.i18n; +using sly.lexer; + +namespace simpleExpressionParser; + +public enum GenericExpressionToken +{ + // float number + [Double] DOUBLE = 1, + + // integer + [Int] INT = 3, + + [AlphaNumId] IDENTIFIER = 4, + + // the + operator + [LexemeLabel("en", "plus sign")] [LexemeLabel("fr", "plus")] + [Sugar("+")] + PLUS = 5, + + // the - operator + [LexemeLabel("en", "minus sign")] [LexemeLabel("fr", "moins")] [Sugar("-")] + MINUS = 6, + + // the * operator + [LexemeLabel("en", "times sign")] [LexemeLabel("fr", "multiplication")] [Sugar("*")] + TIMES = 7, + + // the / operator + [LexemeLabel("en", "divide sign")] [LexemeLabel("fr", "division")] [Sugar("/")] + DIVIDE = 8, + + // a left paranthesis ( + [LexemeLabel("en", "opening parenthesis")] [LexemeLabel("fr", "parenthèse ouvrante")] [Sugar("(")] + LPAREN = 9, + + // a right paranthesis ) + [LexemeLabel("en", "closing parenthesis")] [LexemeLabel("fr", "parenthèse fermante")] [Sugar(")")] + RPAREN = 10, + + [LexemeLabel("fr", "point d'exclamation")] [LexemeLabel("en", "exclamation point")] [Sugar("!")] + FACTORIAL = 13, + +} \ No newline at end of file diff --git a/src/samples/SimpleExpressionParser/GenericSimpleExpressionParser.cs b/src/samples/SimpleExpressionParser/GenericSimpleExpressionParser.cs new file mode 100644 index 00000000..502f2391 --- /dev/null +++ b/src/samples/SimpleExpressionParser/GenericSimpleExpressionParser.cs @@ -0,0 +1,106 @@ +using expressionparser; +using sly.lexer; +using sly.parser.generator; + +namespace simpleExpressionParser; + +[ParserRoot("root")] +//[BroadenTokenWindow] +public class GenericSimpleExpressionParser +{ + + [Production("root : GenericSimpleExpressionParser_expressions")] + + public double Root(double value) => value; + + [Operation((int) GenericExpressionToken.PLUS, Affix.InFix, Associativity.Right, 10)] + [Operation("MINUS", Affix.InFix, Associativity.Left, 10)] + public double BinaryTermExpression(double left, Token operation, double right) + { + double result = 0; + switch (operation.TokenID) + { + case GenericExpressionToken.PLUS: + { + result = left + right; + break; + } + case GenericExpressionToken.MINUS: + { + result = left - right; + break; + } + } + + return result; + } + + + [Operation((int) GenericExpressionToken.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)] + [NodeName("multiplication_or_division")] + public double BinaryFactorExpression(double left, Token operation, double right) + { + double result = 0; + switch (operation.TokenID) + { + case GenericExpressionToken.TIMES: + { + result = left * right; + break; + } + case GenericExpressionToken.DIVIDE: + { + result = left / right; + break; + } + } + + return result; + } + + + [Prefix((int) GenericExpressionToken.MINUS, Associativity.Right, 100)] + public double PreFixExpression(Token operation, double value) + { + return -value; + } + + [Postfix((int) GenericExpressionToken.FACTORIAL, Associativity.Right, 100)] + public double PostFixExpression(double value, Token operation) + { + var factorial = 1; + for (var i = 1; i <= value; i++) factorial = factorial * i; + return factorial; + } + + [Operand] + [Production("operand : primary_value")] + [NodeName("double")] + public double OperandValue(double value) + { + return value; + } + + + [Production("primary_value : DOUBLE")] + [NodeName("double")] + public double OperandDouble(Token value) + { + return value.DoubleValue; + } + + [Production("primary_value : INT")] + [NodeName("integer")] + public double OperandInt(Token value) + { + return value.DoubleValue; + } + + [Production("primary_value : LPAREN GenericSimpleExpressionParser_expressions RPAREN")] + [NodeName("group")] + public double OperandParens(Token lparen, double value, Token rparen) + { + return value; + } +} \ No newline at end of file diff --git a/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs b/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs index 49145a52..a52bb024 100644 --- a/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs +++ b/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs @@ -5,10 +5,12 @@ namespace simpleExpressionParser { [ParserRoot("root")] + //[BroadenTokenWindow] public class SimpleExpressionParser { [Production("root : SimpleExpressionParser_expressions")] + public double Root(double value) => value; [Operation((int) ExpressionToken.PLUS, Affix.InFix, Associativity.Right, 10)] @@ -102,4 +104,7 @@ public double OperandParens(Token lparen, double value, Token ParseWithContext(IList> tokens, object par else { result.Errors = new List(); - var unexpectedTokens = syntaxResult.Errors.ToList(); + var unexpectedTokens = syntaxResult.GetErrors(); var byEnding = unexpectedTokens.GroupBy(x => x.UnexpectedToken.Position).OrderBy(x => x.Key); var errors = new List(); foreach (var expecting in byEnding) diff --git a/src/sly/parser/parser/SyntaxParseResult.cs b/src/sly/parser/parser/SyntaxParseResult.cs index 0cad940f..695d328c 100644 --- a/src/sly/parser/parser/SyntaxParseResult.cs +++ b/src/sly/parser/parser/SyntaxParseResult.cs @@ -14,34 +14,57 @@ public class SyntaxParseResult where IN : struct public bool IsOk => !IsError; - public List> Errors { get; set; } = new List>(); + + + private List> Errors { get; set; } //= new List>(); public int EndingPosition { get; set; } public bool IsEnded { get; set; } + + private void InitErrors() + { + if (Errors == null) { + Errors = new List>(); + } + } + + public void AddErrors(IList> errors) + { + InitErrors(); + Errors.AddRange(errors); + } + + public void AddError(UnexpectedTokenSyntaxError error) + { + InitErrors(); + Errors.Add(error); + } + + public IList> GetErrors() => Errors; public List> Expecting {get; set;} public void AddExpecting(LeadingToken expected) { - if (Expecting == null) - { - Expecting = new List>(); - } - Expecting.Add(expected); + // if (Expecting == null) + // { + // Expecting = new List>(); + // } + // Expecting.Add(expected); } public void AddExpectings(IEnumerable> expected) { - if (expected == null) - { - return; - } - if (Expecting == null) - { - Expecting = new List>(); - } - Expecting.AddRange(expected); + // if (expected == null) + // { + // return; + // } + // if (Expecting == null) + // { + // Expecting = new List>(); + // } + // Expecting.AddRange(expected); } public bool HasByPassNodes { get; set; } = false; diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs index ed9a3beb..7a15c6f1 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs @@ -45,14 +45,15 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, string no rulesResults.Add(innerRuleRes); var other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0; - if (innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.Errors != null && - innerRuleRes.Errors.Count == 0 || other) + if (innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.GetErrors() != null && + innerRuleRes.GetErrors().Count == 0 || other) { greaterIndex = innerRuleRes.EndingPosition; - innerRuleErrors.AddRange(innerRuleRes.Errors); + if (innerRuleRes.GetErrors() != null) + innerRuleErrors.AddRange(innerRuleRes.GetErrors()); } - - innerRuleErrors.AddRange(innerRuleRes.Errors); + if (innerRuleRes.GetErrors() != null) + innerRuleErrors.AddRange(innerRuleRes.GetErrors()); } i++; @@ -109,7 +110,7 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, string no var result = new SyntaxParseResult(); - result.Errors = errors; + result.AddErrors(errors); result.Root = max.Root; result.EndingPosition = max.EndingPosition; result.IsError = max.IsError; @@ -120,8 +121,11 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, string no List> terr = new List>(); foreach (var ruleResult in rulesResults) { - terr.AddRange(ruleResult.Errors); - result.AddExpectings(ruleResult.Errors.SelectMany(x => x.ExpectedTokens)); + if (ruleResult.GetErrors() != null) + { + terr.AddRange(ruleResult.GetErrors()); + result.AddExpectings(ruleResult.GetErrors().SelectMany(x => x.ExpectedTokens)); + } } parsingContext.Memoize(new NonTerminalClause(nonTerminalName), currentPosition, result); diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs index ce174426..6d40471e 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs @@ -27,8 +27,8 @@ public SyntaxParseResult ParseTerminal(IList> tokens, TerminalClau result.HasByPassNodes = false; if (result.IsError) { - result.Errors.Add( - new UnexpectedTokenSyntaxError(token, LexemeLabels, I18n, terminal.ExpectedToken)); + result.AddError(new UnexpectedTokenSyntaxError(token, LexemeLabels, I18n, terminal.ExpectedToken)); + result.AddExpecting(terminal.ExpectedToken); } diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs index 84e7109e..bc44dec9 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs @@ -87,7 +87,7 @@ public SyntaxParseResult SafeParse(IList> tokens, SyntaxParsingCon if (r.EndingPosition == lastPosition) { furtherResults.Add(r); - errors.AddRange(r.Errors); + errors.AddRange(r.GetErrors()); } } @@ -113,11 +113,11 @@ public SyntaxParseResult SafeParse(IList> tokens, SyntaxParsingCon .Where(e => e.UnexpectedToken.PositionInTokenFlow == lastErrorPosition) .ToList(); - result.Errors = lastErrors; + result.AddErrors(lastErrors); } else { - result.Errors = errors; + result.AddErrors(errors); } result.IsError = true; @@ -167,12 +167,12 @@ public virtual SyntaxParseResult Parse(IList> tokens, Rule rul { children.Add(nonTerminalResult.Root); currentPosition = nonTerminalResult.EndingPosition; - if (nonTerminalResult.Errors != null && nonTerminalResult.Errors.Count > 0) - errors.AddRange(nonTerminalResult.Errors); + if (nonTerminalResult.GetErrors() != null && nonTerminalResult.GetErrors().Count > 0) + errors.AddRange(nonTerminalResult.GetErrors()); } else { - errors.AddRange(nonTerminalResult.Errors); + errors.AddRange(nonTerminalResult.GetErrors()); } isError = nonTerminalResult.IsError; @@ -186,7 +186,7 @@ public virtual SyntaxParseResult Parse(IList> tokens, Rule rul var result = new SyntaxParseResult(); result.IsError = isError; - result.Errors = errors; + result.AddErrors(errors); result.EndingPosition = currentPosition; if (!isError) { @@ -228,7 +228,7 @@ private SyntaxParseResult NoMatchingRuleError(IList> tokens, int c error.IsError = true; error.Root = null; error.IsEnded = false; - error.Errors = noRuleErrors; + error.AddErrors(noRuleErrors); error.EndingPosition = currentPosition; error.Expecting = allAcceptableTokens; diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs index 5c2f949f..540b5655 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs @@ -61,7 +61,7 @@ public SyntaxParseResult ParseChoice(IList> tokens, ChoiceClause>(); var expected = terminalAlternates.Select(x => x.ExpectedToken).ToList(); - result.Errors.Add(new UnexpectedTokenSyntaxError(tokens[currentPosition], LexemeLabels, I18n, + result.AddError(new UnexpectedTokenSyntaxError(tokens[currentPosition], LexemeLabels, I18n, expected.ToArray())); } diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs index c77ae2f6..f77ac057 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs @@ -15,7 +15,6 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList> t string nonTerminalName, SyntaxParsingContext parsingContext) { var currentPosition = position; - var errors = new List>(); var isError = false; var children = new List>(); if (!tokens[position].IsEOS && rule.Match(tokens, position, Configuration) && rule.Clauses != null && @@ -46,7 +45,7 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList> t { if (firstResult.Root is SyntaxNode) { - firstResult.Errors.AddRange(secondResult.Errors); + firstResult.AddErrors(secondResult.GetErrors()); firstResult.AddExpectings(secondResult.Expecting); return firstResult; } @@ -66,7 +65,7 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList> t { if (firstResult.Root is SyntaxNode) { - firstResult.Errors.AddRange(secondResult.Errors); + firstResult.AddErrors(secondResult.GetErrors()); firstResult.AddExpectings(secondResult.Expecting); return firstResult; } @@ -111,7 +110,6 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(IList> t var result = new SyntaxParseResult(); result.IsError = false; - result.Errors = errors; result.EndingPosition = currentPosition; SyntaxNode node = null; diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs index 3843a03a..d074bb17 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs @@ -66,13 +66,16 @@ public SyntaxParseResult ParseZeroOrMore(IList> tokens, ZeroOrMore currentPosition = innerResult.EndingPosition; lastInnerResult = innerResult; hasByPasNodes = hasByPasNodes || innerResult.HasByPassNodes; - innerErrors.AddRange(lastInnerResult.Errors); + if (lastInnerResult.GetErrors() != null) + { + innerErrors.AddRange(lastInnerResult.GetErrors()); + } } else { if (innerResult != null) { - innerErrors.AddRange(innerResult.Errors); + innerErrors.AddRange(innerResult.GetErrors()); } } @@ -82,7 +85,7 @@ public SyntaxParseResult ParseZeroOrMore(IList> tokens, ZeroOrMore result.EndingPosition = currentPosition; result.IsError = false; - result.Errors = innerErrors; + result.AddErrors(innerErrors); result.Root = manyNode; result.IsEnded = lastInnerResult != null && lastInnerResult.IsEnded; result.HasByPassNodes = hasByPasNodes; @@ -153,20 +156,20 @@ public SyntaxParseResult ParseOneOrMore(IList> tokens, OneOrMoreCl if (nextResult != null) { - innerErrors.AddRange(nextResult.Errors); + innerErrors.AddRange(nextResult.GetErrors()); } isError = false; } else { - innerErrors.AddRange(firstInnerResult.Errors); + innerErrors.AddRange(firstInnerResult.GetErrors()); isError = true; } result.EndingPosition = currentPosition; result.IsError = isError; - result.Errors = innerErrors; + result.AddErrors(innerErrors); result.Root = manyNode; result.IsEnded = lastInnerResult != null && lastInnerResult.IsEnded; result.HasByPassNodes = hasByPasNodes; diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs index a738889b..e6f48d83 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs @@ -63,13 +63,13 @@ public override SyntaxParseResult Parse(IList> tokens, Rule ru ParseNonTerminal(tokens, terminalClause, currentPosition, parsingContext); if (!nonTerminalResult.IsError) { - errors.AddRange(nonTerminalResult.Errors); + errors.AddRange(nonTerminalResult.GetErrors()); children.Add(nonTerminalResult.Root); currentPosition = nonTerminalResult.EndingPosition; } else { - errors.AddRange(nonTerminalResult.Errors); + errors.AddRange(nonTerminalResult.GetErrors()); } isError = isError || nonTerminalResult.IsError; @@ -91,14 +91,14 @@ public override SyntaxParseResult Parse(IList> tokens, Rule ru if (!manyResult.IsError) { - errors.AddRange(manyResult.Errors); + errors.AddRange(manyResult.GetErrors()); children.Add(manyResult.Root); currentPosition = manyResult.EndingPosition; } else { - if (manyResult.Errors != null && manyResult.Errors.Count > 0) - errors.AddRange(manyResult.Errors); + if (manyResult.GetErrors() != null && manyResult.GetErrors().Count > 0) + errors.AddRange(manyResult.GetErrors()); } isError = manyResult.IsError; @@ -115,9 +115,9 @@ public override SyntaxParseResult Parse(IList> tokens, Rule ru { var choiceResult = ParseChoice(tokens, choice, currentPosition, parsingContext); currentPosition = choiceResult.EndingPosition; - if (choiceResult.IsError && choiceResult.Errors != null && choiceResult.Errors.Any()) + if (choiceResult.IsError && choiceResult.GetErrors() != null && choiceResult.GetErrors().Any()) { - errors.AddRange(choiceResult.Errors); + errors.AddRange(choiceResult.GetErrors()); } isError = choiceResult.IsError; @@ -133,7 +133,7 @@ public override SyntaxParseResult Parse(IList> tokens, Rule ru var result = new SyntaxParseResult(); result.IsError = isError; - result.Errors = errors; + result.AddErrors(errors); result.EndingPosition = currentPosition; if (!isError) { diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index 72ba1d75..7162bef4 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -159,7 +159,20 @@ public bool Match(IList> tokens, int position, ParserConfiguratio return false; } - return PossibleLeadingTokens.Exists(x => x.Match(tokens[position])) || MayBeEmpty; + int i = 0; + bool match = false; + var token = tokens[position]; + if (MayBeEmpty) + { + return true; + } + while (i < PossibleLeadingTokens.Count && !match) + { + var leader = PossibleLeadingTokens[i]; + match = leader.Match(token); + i++; + } + return match; } [ExcludeFromCodeCoverage] diff --git a/src/sly/sly.csproj b/src/sly/sly.csproj index f76e40a1..7057452e 100644 --- a/src/sly/sly.csproj +++ b/src/sly/sly.csproj @@ -6,10 +6,10 @@ $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb #LY is a parser generator halfway between parser combinators and parser generator like ANTLR b3b00 - 3.2.8 + 3.2.9-alpha1 https://github.com/b3b00/sly https://github.com/b3b00/sly - 3.2.8 + 3.2.9-alpha1 Library diff --git a/tests/ParserTests/EBNFTests.cs b/tests/ParserTests/EBNFTests.cs index a3825009..e3f67f12 100644 --- a/tests/ParserTests/EBNFTests.cs +++ b/tests/ParserTests/EBNFTests.cs @@ -16,6 +16,7 @@ using sly.parser.parser; using sly.parser.syntax.grammar; using Xunit; +using ExpressionToken = simpleExpressionParser.ExpressionToken; using String = System.String; namespace ParserTests diff --git a/tests/ParserTests/ExpressionGeneratorTests.cs b/tests/ParserTests/ExpressionGeneratorTests.cs index 3adf2718..e1d4acbc 100644 --- a/tests/ParserTests/ExpressionGeneratorTests.cs +++ b/tests/ParserTests/ExpressionGeneratorTests.cs @@ -8,6 +8,7 @@ using sly.parser.generator; using Xunit; using ParserTests.Issue184; +using ExpressionToken = expressionparser.ExpressionToken; namespace ParserTests { @@ -163,7 +164,7 @@ public double OperandGroup(Token lparen, double value, Token> Parser; + private BuildResult> Parser; private string StartingRule = ""; @@ -172,7 +173,7 @@ private void BuildParser() { StartingRule = $"{nameof(SimpleExpressionParser)}_expressions"; var parserInstance = new SimpleExpressionParser(); - var builder = new ParserBuilder(); + var builder = new ParserBuilder(); Parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule); } @@ -223,7 +224,7 @@ public void TestBuild() BuildParser(); Check.That(Parser.Result.Configuration.NonTerminals).CountIs(7); - var nonterminals = new List>(); + var nonterminals = new List>(); foreach (var pair in Parser.Result.Configuration.NonTerminals) nonterminals.Add(pair.Value); var nt = nonterminals[1]; // operand Check.That(nt.Rules).IsSingle(); diff --git a/tests/ParserTests/GraphVizTests.cs b/tests/ParserTests/GraphVizTests.cs index d8ef00bf..5212014c 100644 --- a/tests/ParserTests/GraphVizTests.cs +++ b/tests/ParserTests/GraphVizTests.cs @@ -5,6 +5,7 @@ using sly.parser.generator.visitor; using sly.parser.generator.visitor.dotgraph; using Xunit; +using ExpressionToken = simpleExpressionParser.ExpressionToken; namespace ParserTests { diff --git a/tests/ParserTests/Issue164/Issue164Tests.cs b/tests/ParserTests/Issue164/Issue164Tests.cs index aef383c6..d2f361bd 100644 --- a/tests/ParserTests/Issue164/Issue164Tests.cs +++ b/tests/ParserTests/Issue164/Issue164Tests.cs @@ -5,6 +5,7 @@ using sly.parser; using sly.parser.generator; using Xunit; +using ExpressionToken = simpleExpressionParser.ExpressionToken; namespace ParserTests.Issue164 { diff --git a/tests/ParserTests/Issue380I18n/I18nTests.cs b/tests/ParserTests/Issue380I18n/I18nTests.cs index 62044bea..2d2d7bfa 100644 --- a/tests/ParserTests/Issue380I18n/I18nTests.cs +++ b/tests/ParserTests/Issue380I18n/I18nTests.cs @@ -5,6 +5,7 @@ using sly.parser; using sly.parser.generator; using Xunit; +using ExpressionToken = simpleExpressionParser.ExpressionToken; namespace ParserTests.Issue380I18n; diff --git a/tests/ParserTests/lexer/GenericLexerTests.cs b/tests/ParserTests/lexer/GenericLexerTests.cs index cd274425..3586d139 100644 --- a/tests/ParserTests/lexer/GenericLexerTests.cs +++ b/tests/ParserTests/lexer/GenericLexerTests.cs @@ -13,6 +13,7 @@ using sly.parser; using sly.parser.generator; using Xunit; +using ExpressionToken = simpleExpressionParser.ExpressionToken; namespace ParserTests.lexer {