From 92ad8f4de9191a0b20d788e50126736d584318ed Mon Sep 17 00:00:00 2001 From: Roman Ivantsov <594969+rivantsov@users.noreply.github.com> Date: Tue, 3 Jan 2012 07:40:28 -0800 Subject: [PATCH] finish refactoring of AST build process --- Irony.GrammarExplorer/fmGrammarExplorer.cs | 8 +- .../Ast/AstContext/InterpreterAstContext.cs | 5 +- .../Ast/Expressions/BinaryOperationNode.cs | 7 +- .../Ast/Expressions/ExpressionListNode.cs | 2 +- Irony.Interpreter/Ast/Expressions/IfNode.cs | 9 +- .../Ast/Expressions/IncDecNode.cs | 11 +- .../Ast/Expressions/IndexedAccessNode.cs | 5 +- .../Ast/Expressions/MemberAccessNode.cs | 5 +- .../Ast/Expressions/UnaryOperationNode.cs | 5 +- .../Ast/Functions/FunctionCallNode.cs | 7 +- .../Ast/Functions/FunctionDefNode.cs | 7 +- .../Ast/Functions/ParamListNode.cs | 2 +- .../Ast/PrimitiveNodes/StringTemplateNode.cs | 2 +- .../Ast/Statements/AssignmentNode.cs | 7 +- .../Ast/Statements/StatementListNode.cs | 3 +- .../InterpretedLanguageGrammar.cs | 2 - .../_Evaluator/ExpressionEvaluatorGrammar.cs | 4 +- .../040.Irony.Tests.VsTest.2010.csproj | 1 + Irony.Tests/ErrorRecoveryTests.cs | 57 +++++ .../ConflictResolutionTests.cs | 12 +- Irony/010.Irony.2010.csproj | 4 +- Irony/101.IronySilverlight.2010.csproj | 3 +- Irony/Ast/AstBuilder.cs | 129 +++++++----- Irony/Ast/AstContext.cs | 20 +- Irony/Ast/AstExtensions.cs | 29 +++ Irony/Ast/AstInterfaces.cs | 17 +- Irony/Ast/AstNodeConfig.cs | 67 +++--- .../ConditionalParserAction.cs | 2 +- .../CustomActionHintAction.cs | 14 +- .../Data/Construction/GrammarDataBuilder.cs | 9 +- Irony/Parsing/Data/LanguageData.cs | 1 + Irony/Parsing/Grammar/BnfTerm.cs | 58 ++---- Irony/Parsing/Grammar/Grammar.cs | 4 - Irony/Parsing/Parser/CoreParser.cs | 195 ------------------ Irony/Parsing/Parser/ParseTree.cs | 31 +-- Irony/Parsing/Parser/ParseTreeExtensions.cs | 5 +- Irony/Parsing/Parser/Parser.cs | 161 ++++++++++++++- .../ErrorRecoveryParserAction.cs | 20 +- Irony/Parsing/Scanner/SourceStream.cs | 2 +- Irony/Parsing/Terminals/ConstantTerminal.cs | 2 +- Irony/Parsing/Terminals/IdentifierTerminal.cs | 3 - Irony/Parsing/Terminals/NumberLiteral.cs | 5 +- Irony/Parsing/Terminals/StringLiteral.cs | 8 +- Irony/Parsing/Terminals/_Terminal.cs | 5 - Irony/Resources.Designer.cs | 31 ++- Irony/Resources.resx | 13 +- 46 files changed, 545 insertions(+), 454 deletions(-) create mode 100644 Irony.Tests/ErrorRecoveryTests.cs create mode 100644 Irony/Ast/AstExtensions.cs delete mode 100644 Irony/Parsing/Parser/CoreParser.cs diff --git a/Irony.GrammarExplorer/fmGrammarExplorer.cs b/Irony.GrammarExplorer/fmGrammarExplorer.cs index 03a04c4..eed4f98 100644 --- a/Irony.GrammarExplorer/fmGrammarExplorer.cs +++ b/Irony.GrammarExplorer/fmGrammarExplorer.cs @@ -188,10 +188,10 @@ private void ShowParserConstructionResults() { txtNonTerms.Text = string.Empty; txtParserStates.Text = string.Empty; tabBottom.SelectedTab = pageLanguage; - if (_parser == null) return; - txtTerms.Text = ParserDataPrinter.PrintTerminals(_parser.Language); - txtNonTerms.Text = ParserDataPrinter.PrintNonTerminals(_parser.Language); - txtParserStates.Text = ParserDataPrinter.PrintStateList(_parser.Language); + if (_parser == null) return; + txtTerms.Text = ParserDataPrinter.PrintTerminals(_language); + txtNonTerms.Text = ParserDataPrinter.PrintNonTerminals(_language); + txtParserStates.Text = ParserDataPrinter.PrintStateList(_language); ShowGrammarErrors(); }//method diff --git a/Irony.Interpreter/Ast/AstContext/InterpreterAstContext.cs b/Irony.Interpreter/Ast/AstContext/InterpreterAstContext.cs index 4a31347..df6f0cb 100644 --- a/Irony.Interpreter/Ast/AstContext/InterpreterAstContext.cs +++ b/Irony.Interpreter/Ast/AstContext/InterpreterAstContext.cs @@ -12,7 +12,10 @@ public class InterpreterAstContext : AstContext { public readonly OperatorHandler OperatorHandler; public InterpreterAstContext(LanguageData language, OperatorHandler operatorHandler = null) : base(language) { - OperatorHandler = operatorHandler ?? new OperatorHandler(language.Grammar.CaseSensitive); + OperatorHandler = operatorHandler ?? new OperatorHandler(language.Grammar.CaseSensitive); + base.DefaultIdentifierNodeType = typeof(IdentifierNode); + base.DefaultLiteralNodeType = typeof(LiteralValueNode); + base.DefaultNodeType = null; } }//class diff --git a/Irony.Interpreter/Ast/Expressions/BinaryOperationNode.cs b/Irony.Interpreter/Ast/Expressions/BinaryOperationNode.cs index a709765..4b5687a 100644 --- a/Irony.Interpreter/Ast/Expressions/BinaryOperationNode.cs +++ b/Irony.Interpreter/Ast/Expressions/BinaryOperationNode.cs @@ -33,9 +33,10 @@ public BinaryOperationNode() { } public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - Left = AddChild("Arg", treeNode.MappedChildNodes[0]); - Right = AddChild("Arg", treeNode.MappedChildNodes[2]); - var opToken = treeNode.MappedChildNodes[1].FindToken(); + var nodes = treeNode.GetMappedChildNodes(); + Left = AddChild("Arg", nodes[0]); + Right = AddChild("Arg", nodes[2]); + var opToken = nodes[1].FindToken(); OpSymbol = opToken.Text; var ictxt = context as InterpreterAstContext; Op = ictxt.OperatorHandler.GetOperatorExpressionType(OpSymbol); diff --git a/Irony.Interpreter/Ast/Expressions/ExpressionListNode.cs b/Irony.Interpreter/Ast/Expressions/ExpressionListNode.cs index dc6647c..ce0916f 100644 --- a/Irony.Interpreter/Ast/Expressions/ExpressionListNode.cs +++ b/Irony.Interpreter/Ast/Expressions/ExpressionListNode.cs @@ -25,7 +25,7 @@ public class ExpressionListNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - foreach (var child in treeNode.MappedChildNodes) { + foreach (var child in treeNode.ChildNodes) { AddChild(NodeUseType.Parameter, "expr", child); } AsString = "Expression list"; diff --git a/Irony.Interpreter/Ast/Expressions/IfNode.cs b/Irony.Interpreter/Ast/Expressions/IfNode.cs index 0a2c72e..6eb85c0 100644 --- a/Irony.Interpreter/Ast/Expressions/IfNode.cs +++ b/Irony.Interpreter/Ast/Expressions/IfNode.cs @@ -25,10 +25,11 @@ public class IfNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - Test = AddChild("Test", treeNode.MappedChildNodes[0]); - IfTrue = AddChild("IfTrue", treeNode.MappedChildNodes[1]); - if (treeNode.MappedChildNodes.Count > 2) - IfFalse = AddChild("IfFalse", treeNode.MappedChildNodes[2]); + var nodes = treeNode.GetMappedChildNodes(); + Test = AddChild("Test", nodes[0]); + IfTrue = AddChild("IfTrue", nodes[1]); + if (nodes.Count > 2) + IfFalse = AddChild("IfFalse", nodes[2]); } protected override object DoEvaluate(ScriptThread thread) { diff --git a/Irony.Interpreter/Ast/Expressions/IncDecNode.cs b/Irony.Interpreter/Ast/Expressions/IncDecNode.cs index 8aeff47..0af17d1 100644 --- a/Irony.Interpreter/Ast/Expressions/IncDecNode.cs +++ b/Irony.Interpreter/Ast/Expressions/IncDecNode.cs @@ -31,21 +31,22 @@ public class IncDecNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - FindOpAndDetectPostfix(treeNode); + var nodes = treeNode.GetMappedChildNodes(); + FindOpAndDetectPostfix(nodes); int argIndex = IsPostfix? 0 : 1; - Argument = AddChild(NodeUseType.ValueReadWrite, "Arg", treeNode.MappedChildNodes[argIndex]); + Argument = AddChild(NodeUseType.ValueReadWrite, "Arg", nodes[argIndex]); BinaryOpSymbol = OpSymbol[0].ToString(); //take a single char out of ++ or -- var interpContext = (InterpreterAstContext)context; BinaryOp = interpContext.OperatorHandler.GetOperatorExpressionType(BinaryOpSymbol); base.AsString = OpSymbol + (IsPostfix ? "(postfix)" : "(prefix)"); } - private void FindOpAndDetectPostfix(ParseTreeNode treeNode) { + private void FindOpAndDetectPostfix(ParseTreeNodeList mappedNodes) { IsPostfix = false; //assume it - OpSymbol = treeNode.MappedChildNodes[0].FindTokenAndGetText(); + OpSymbol = mappedNodes[0].FindTokenAndGetText(); if (OpSymbol == "--" || OpSymbol == "++") return; IsPostfix = true; - OpSymbol = treeNode.MappedChildNodes[1].FindTokenAndGetText(); + OpSymbol = mappedNodes[1].FindTokenAndGetText(); } protected override object DoEvaluate(ScriptThread thread) { diff --git a/Irony.Interpreter/Ast/Expressions/IndexedAccessNode.cs b/Irony.Interpreter/Ast/Expressions/IndexedAccessNode.cs index b9939b1..e01e56d 100644 --- a/Irony.Interpreter/Ast/Expressions/IndexedAccessNode.cs +++ b/Irony.Interpreter/Ast/Expressions/IndexedAccessNode.cs @@ -15,8 +15,9 @@ public class IndexedAccessNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - _target = AddChild("Target", treeNode.FirstChild); - _index = AddChild("Index", treeNode.LastChild); + var nodes = treeNode.GetMappedChildNodes(); + _target = AddChild("Target", nodes.First()); + _index = AddChild("Index", nodes.Last()); AsString = "[" + _index + "]"; } diff --git a/Irony.Interpreter/Ast/Expressions/MemberAccessNode.cs b/Irony.Interpreter/Ast/Expressions/MemberAccessNode.cs index 3449479..49b58a5 100644 --- a/Irony.Interpreter/Ast/Expressions/MemberAccessNode.cs +++ b/Irony.Interpreter/Ast/Expressions/MemberAccessNode.cs @@ -17,8 +17,9 @@ public class MemberAccessNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - _left = AddChild("Target", treeNode.FirstChild); - var right = treeNode.LastChild; + var nodes = treeNode.GetMappedChildNodes(); + _left = AddChild("Target", nodes[0]); + var right = nodes[nodes.Count - 1]; _memberName = right.FindTokenAndGetText(); ErrorAnchor = right.Span.Location; AsString = "." + _memberName; diff --git a/Irony.Interpreter/Ast/Expressions/UnaryOperationNode.cs b/Irony.Interpreter/Ast/Expressions/UnaryOperationNode.cs index eafca27..e3d99a9 100644 --- a/Irony.Interpreter/Ast/Expressions/UnaryOperationNode.cs +++ b/Irony.Interpreter/Ast/Expressions/UnaryOperationNode.cs @@ -27,8 +27,9 @@ public class UnaryOperationNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - OpSymbol = treeNode.MappedChildNodes[0].FindTokenAndGetText(); - Argument = AddChild("Arg", treeNode.MappedChildNodes[1]); + var nodes = treeNode.GetMappedChildNodes(); + OpSymbol = nodes[0].FindTokenAndGetText(); + Argument = AddChild("Arg", nodes[1]); base.AsString = OpSymbol + "(unary op)"; var interpContext = (InterpreterAstContext)context; base.ExpressionType = interpContext.OperatorHandler.GetUnaryOperatorExpressionType(OpSymbol); diff --git a/Irony.Interpreter/Ast/Functions/FunctionCallNode.cs b/Irony.Interpreter/Ast/Functions/FunctionCallNode.cs index 2bffdbd..e3d05b7 100644 --- a/Irony.Interpreter/Ast/Functions/FunctionCallNode.cs +++ b/Irony.Interpreter/Ast/Functions/FunctionCallNode.cs @@ -30,10 +30,11 @@ public class FunctionCallNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - TargetRef = AddChild("Target", treeNode.MappedChildNodes[0]); + var nodes = treeNode.GetMappedChildNodes(); + TargetRef = AddChild("Target", nodes[0]); TargetRef.UseType = NodeUseType.CallTarget; - _targetName = treeNode.MappedChildNodes[0].FindTokenAndGetText(); - Arguments = AddChild("Args", treeNode.MappedChildNodes[1]); + _targetName = nodes[0].FindTokenAndGetText(); + Arguments = AddChild("Args", nodes[1]); AsString = "Call " + _targetName; } diff --git a/Irony.Interpreter/Ast/Functions/FunctionDefNode.cs b/Irony.Interpreter/Ast/Functions/FunctionDefNode.cs index b65fb00..7a36354 100644 --- a/Irony.Interpreter/Ast/Functions/FunctionDefNode.cs +++ b/Irony.Interpreter/Ast/Functions/FunctionDefNode.cs @@ -30,9 +30,10 @@ public class FunctionDefNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); //child #0 is usually a keyword like "def" - NameNode = AddChild("Name", treeNode.MappedChildNodes[1]); - Parameters = AddChild("Parameters", treeNode.MappedChildNodes[2]); - Body = AddChild("Body", treeNode.MappedChildNodes[3]); + var nodes = treeNode.GetMappedChildNodes(); + NameNode = AddChild("Name", nodes[1]); + Parameters = AddChild("Parameters", nodes[2]); + Body = AddChild("Body", nodes[3]); AsString = ""; Body.SetIsTail(); //this will be propagated to the last statement } diff --git a/Irony.Interpreter/Ast/Functions/ParamListNode.cs b/Irony.Interpreter/Ast/Functions/ParamListNode.cs index 73e0855..ed8290e 100644 --- a/Irony.Interpreter/Ast/Functions/ParamListNode.cs +++ b/Irony.Interpreter/Ast/Functions/ParamListNode.cs @@ -24,7 +24,7 @@ public class ParamListNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - foreach (var child in treeNode.MappedChildNodes) + foreach (var child in treeNode.ChildNodes) AddChild(NodeUseType.Parameter, "param", child); AsString = "param_list[" + ChildNodes.Count + "]"; } diff --git a/Irony.Interpreter/Ast/PrimitiveNodes/StringTemplateNode.cs b/Irony.Interpreter/Ast/PrimitiveNodes/StringTemplateNode.cs index e2b9c0b..0109de6 100644 --- a/Irony.Interpreter/Ast/PrimitiveNodes/StringTemplateNode.cs +++ b/Irony.Interpreter/Ast/PrimitiveNodes/StringTemplateNode.cs @@ -68,7 +68,7 @@ public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); _template = treeNode.Token.ValueString; _tokenText = treeNode.Token.Text; - _templateSettings = treeNode.Term.AstData as StringTemplateSettings; + _templateSettings = treeNode.Term.AstConfig.Data as StringTemplateSettings; ParseSegments(context); AsString = "\"" + _template + "\" (templated string)"; } diff --git a/Irony.Interpreter/Ast/Statements/AssignmentNode.cs b/Irony.Interpreter/Ast/Statements/AssignmentNode.cs index 86a7326..7c59ec5 100644 --- a/Irony.Interpreter/Ast/Statements/AssignmentNode.cs +++ b/Irony.Interpreter/Ast/Statements/AssignmentNode.cs @@ -30,14 +30,15 @@ public class AssignmentNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - Target = AddChild(NodeUseType.ValueWrite, "To", treeNode.MappedChildNodes[0]); + var nodes = treeNode.GetMappedChildNodes(); + Target = AddChild(NodeUseType.ValueWrite, "To", nodes[0]); //Get Op and baseOp if it is combined assignment - AssignmentOp = treeNode.MappedChildNodes[1].FindTokenAndGetText(); + AssignmentOp = nodes[1].FindTokenAndGetText(); if (string.IsNullOrEmpty(AssignmentOp)) AssignmentOp = "="; BinaryExpressionType = CustomExpressionTypes.NotAnExpression; //There maybe an "=" sign in the middle, or not - if it is marked as punctuation; so we just take the last node in child list - Expression = AddChild(NodeUseType.ValueRead, "Expr", treeNode.LastChild); + Expression = AddChild(NodeUseType.ValueRead, "Expr", nodes[nodes.Count - 1]); AsString = AssignmentOp + " (assignment)"; // TODO: this is not always correct: in Pascal the assignment operator is :=. IsAugmented = AssignmentOp.Length > 1; diff --git a/Irony.Interpreter/Ast/Statements/StatementListNode.cs b/Irony.Interpreter/Ast/Statements/StatementListNode.cs index 86c4233..2de0fff 100644 --- a/Irony.Interpreter/Ast/Statements/StatementListNode.cs +++ b/Irony.Interpreter/Ast/Statements/StatementListNode.cs @@ -25,7 +25,8 @@ public class StatementListNode : AstNode { public override void Init(AstContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); - foreach (var child in treeNode.MappedChildNodes) { + var nodes = treeNode.GetMappedChildNodes(); + foreach (var child in nodes) { //don't add if it is null; it can happen that "statement" is a comment line and statement's node is null. // So to make life easier for language creator, we just skip if it is null if (child.AstNode != null) diff --git a/Irony.Interpreter/InterpretedLanguageGrammar.cs b/Irony.Interpreter/InterpretedLanguageGrammar.cs index f85ac47..f3f9817 100644 --- a/Irony.Interpreter/InterpretedLanguageGrammar.cs +++ b/Irony.Interpreter/InterpretedLanguageGrammar.cs @@ -25,8 +25,6 @@ public abstract class InterpretedLanguageGrammar : Grammar, ICanRunSample { // making the class abstract so it won't load into Grammar Explorer public InterpretedLanguageGrammar(bool caseSensitive) : base(caseSensitive) { - this.DefaultLiteralNodeType = typeof(LiteralValueNode); //default node type for literals - this.DefaultIdentifierNodeType = typeof(IdentifierNode); this.LanguageFlags = LanguageFlags.CreateAst; } diff --git a/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs b/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs index 6f1bf05..4575c87 100644 --- a/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs +++ b/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs @@ -51,12 +51,12 @@ public ExpressionEvaluatorGrammar() : base(caseSensitive: false) { //String literal with embedded expressions ------------------------------------------------------------------ var stringLit = new StringLiteral("string", "\"", StringOptions.AllowsAllEscapes | StringOptions.IsTemplate); stringLit.AddStartEnd("'", StringOptions.AllowsAllEscapes | StringOptions.IsTemplate); - stringLit.AstNodeType = typeof(StringTemplateNode); + stringLit.AstConfig.NodeType = typeof(StringTemplateNode); var Expr = new NonTerminal("Expr"); //declare it here to use in template definition var templateSettings = new StringTemplateSettings(); //by default set to Ruby-style settings templateSettings.ExpressionRoot = Expr; //this defines how to evaluate expressions inside template this.SnippetRoots.Add(Expr); - stringLit.AstData = templateSettings; + stringLit.AstConfig.Data = templateSettings; //-------------------------------------------------------------------------------------------------------- // 2. Non-terminals diff --git a/Irony.Tests/040.Irony.Tests.VsTest.2010.csproj b/Irony.Tests/040.Irony.Tests.VsTest.2010.csproj index ad68e40..b9999a4 100644 --- a/Irony.Tests/040.Irony.Tests.VsTest.2010.csproj +++ b/Irony.Tests/040.Irony.Tests.VsTest.2010.csproj @@ -71,6 +71,7 @@ + diff --git a/Irony.Tests/ErrorRecoveryTests.cs b/Irony.Tests/ErrorRecoveryTests.cs new file mode 100644 index 0000000..421587b --- /dev/null +++ b/Irony.Tests/ErrorRecoveryTests.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.RegularExpressions; +using Irony.Parsing; + +namespace Irony.Tests { +#if USE_NUNIT + using NUnit.Framework; + using TestClass = NUnit.Framework.TestFixtureAttribute; + using TestMethod = NUnit.Framework.TestAttribute; + using TestInitialize = NUnit.Framework.SetUpAttribute; +#else + using Microsoft.VisualStudio.TestTools.UnitTesting; +#endif + + [TestClass] + public class ErrorRecoveryTests { + + #region Grammars + //A simple grammar for language consisting of simple assignment statements: x=y + z; z= t + m; + public class ErrorRecoveryGrammar : Grammar { + public ErrorRecoveryGrammar() { + var id = new IdentifierTerminal("id"); + var expr = new NonTerminal("expr"); + var stmt = new NonTerminal("stmt"); + var stmtList = new NonTerminal("stmt"); + + base.Root = stmtList; + stmtList.Rule = MakeStarRule(stmtList, stmt); + stmt.Rule = id + "=" + expr + ";"; + stmt.ErrorRule = SyntaxError + ";"; + expr.Rule = id | id + "+" + id; + } + }// class + + #endregion + + [TestMethod] + public void TestErrorRecovery() { + + var grammar = new ErrorRecoveryGrammar(); + var parser = new Parser(grammar); + TestHelper.CheckGrammarErrors(parser); + + //correct sample + var parseTree = parser.Parse("x = y; y = z + m; m = n;"); + Assert.IsFalse(parseTree.HasErrors(), "Unexpected parse errors in correct source sample."); + + parseTree = parser.Parse("x = y; m = = d ; y = z + m; x = z z; m = n;"); + Assert.AreEqual(2, parseTree.ParserMessages.Count, "Invalid # of errors."); + + } + + + }//class +}//namespace diff --git a/Irony.Tests/TokenPreviewResolution/ConflictResolutionTests.cs b/Irony.Tests/TokenPreviewResolution/ConflictResolutionTests.cs index ebca808..e73cbab 100644 --- a/Irony.Tests/TokenPreviewResolution/ConflictResolutionTests.cs +++ b/Irony.Tests/TokenPreviewResolution/ConflictResolutionTests.cs @@ -60,8 +60,8 @@ public void TestConflictGrammarWithHintsOnRules() { Assert.AreEqual("definition", term.Name); Assert.AreEqual(1, tree.Root.ChildNodes.Count); - var nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray(); - Assert.AreEqual("fieldModifier", nodes[0].Term.Name); + var modNode = tree.Root.ChildNodes[0].ChildNodes[0]; + Assert.AreEqual("fieldModifier", modNode.Term.Name); //Property sample = PropertySample; @@ -75,8 +75,8 @@ public void TestConflictGrammarWithHintsOnRules() { Assert.AreEqual("definition", term.Name); Assert.AreEqual(1, tree.Root.ChildNodes.Count); - nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray(); - Assert.AreEqual("propModifier", nodes[0].Term.Name); + modNode = tree.Root.ChildNodes[0].ChildNodes[0]; + Assert.AreEqual("propModifier", modNode.Term.Name); } //Hints on terms --------------------------------------------------------------------- @@ -98,7 +98,7 @@ public void TestConflictGrammar_HintsOnTerms() { Assert.AreEqual("StatementList", term.Name); Assert.AreEqual(2, tree.Root.ChildNodes.Count); - var nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray(); + var nodes = tree.Root.ChildNodes.Select(t => t.ChildNodes[0]).ToArray(); Assert.AreEqual("fieldDef", nodes[0].Term.Name); Assert.AreEqual("fieldDef", nodes[1].Term.Name); @@ -114,7 +114,7 @@ public void TestConflictGrammar_HintsOnTerms() { Assert.AreEqual("StatementList", term.Name); Assert.AreEqual(3, tree.Root.ChildNodes.Count); - nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray(); + nodes = tree.Root.ChildNodes.Select(t => t.ChildNodes[0]).ToArray(); Assert.AreEqual("propDef", nodes[0].Term.Name); Assert.AreEqual("fieldDef", nodes[1].Term.Name); Assert.AreEqual("methodDef", nodes[2].Term.Name); diff --git a/Irony/010.Irony.2010.csproj b/Irony/010.Irony.2010.csproj index 25d8925..5e8657c 100644 --- a/Irony/010.Irony.2010.csproj +++ b/Irony/010.Irony.2010.csproj @@ -76,7 +76,8 @@ - + + @@ -125,7 +126,6 @@ - diff --git a/Irony/101.IronySilverlight.2010.csproj b/Irony/101.IronySilverlight.2010.csproj index 8b56671..12b1b9f 100644 --- a/Irony/101.IronySilverlight.2010.csproj +++ b/Irony/101.IronySilverlight.2010.csproj @@ -58,6 +58,7 @@ + @@ -86,7 +87,6 @@ - @@ -144,7 +144,6 @@ - diff --git a/Irony/Ast/AstBuilder.cs b/Irony/Ast/AstBuilder.cs index 99a2aa7..377677b 100644 --- a/Irony/Ast/AstBuilder.cs +++ b/Irony/Ast/AstBuilder.cs @@ -1,4 +1,16 @@ -using System; +#region License +/* ********************************************************************************** + * Copyright (c) Roman Ivantsov + * This source code is subject to terms and conditions of the MIT License + * for Irony. A copy of the license can be found in the License.txt file + * at the root of this distribution. + * By using this source code in any fashion, you are agreeing to be bound by the terms of the + * MIT License. + * You must not remove this notice from this software. + * **********************************************************************************/ +#endregion + +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -16,40 +28,77 @@ public AstBuilder(AstContext context) { } public virtual void BuildAst(ParseTree parseTree) { - Context.Messages = parseTree.ParserMessages; + if (parseTree.Root == null) + return; + Context.Messages = parseTree.ParserMessages; + if (!Context.Language.AstDataVerified) + VerifyLanguageData(); + if (Context.Language.ErrorLevel == GrammarErrorLevel.Error) + return; BuildAst(parseTree.Root); } + public virtual void VerifyLanguageData() { + var gd = Context.Language.GrammarData; + //Collect all terminals and non-terminals + var terms = new BnfTermSet(); + //SL does not understand co/contravariance, so doing merge one-by-one + foreach (var t in gd.Terminals) terms.Add(t); + foreach (var t in gd.NonTerminals) terms.Add(t); + var missingList = new BnfTermList(); + foreach (var term in terms) { + var terminal = term as Terminal; + if (terminal != null && terminal.Category != TokenCategory.Content) continue; //only content terminals + if (term.Flags.IsSet(TermFlags.NoAstNode)) continue; + var config = term.AstConfig; + if (config.NodeCreator != null || config.DefaultNodeCreator != null) continue; + //We must check NodeType + if (config.NodeType == null) + config.NodeType = GetDefaultNodeType(term); + if (config.NodeType == null) + missingList.Add(term); + else + config.DefaultNodeCreator = CompileDefaultNodeCreator(config.NodeType); + } + if (missingList.Count > 0) + Context.AddMessage(ErrorLevel.Error, SourceLocation.Empty, Resources.ErrNodeTypeNotSetOn, missingList.ToString()); + Context.Language.AstDataVerified = true; + } + // AST node type is not specified for term {0}. Either assign Term.AstConfig.NodeType, or specify default type(s) in AstBuilder. + protected virtual Type GetDefaultNodeType(BnfTerm term) { + if (term is NumberLiteral || term is StringLiteral) + return Context.DefaultLiteralNodeType; + else if (term is IdentifierTerminal) + return Context.DefaultIdentifierNodeType; + else + return Context.DefaultNodeType; + } + public virtual void BuildAst(ParseTreeNode parseNode) { var term = parseNode.Term; if (term.Flags.IsSet(TermFlags.NoAstNode) || parseNode.AstNode != null) return; //children first var processChildren = !parseNode.Term.Flags.IsSet(TermFlags.AstDelayChildren) && parseNode.ChildNodes.Count > 0; if (processChildren) { - var mappedChildNodes = parseNode.MappedChildNodes; + var mappedChildNodes = parseNode.GetMappedChildNodes(); for (int i = 0; i < mappedChildNodes.Count; i++) BuildAst(mappedChildNodes[i]); } //create the node - //First check the custom creator delegate - if (term.AstNodeCreator != null) { - term.AstNodeCreator(Context, parseNode); + //We know that either NodeCreator or DefaultNodeCreator is set; VerifyAstData create the DefaultNodeCreator + var config = term.AstConfig; + if (config.NodeCreator != null) { + config.NodeCreator(Context, parseNode); // We assume that Node creator method creates node and initializes it, so parser does not need to call // IAstNodeInit.Init() method on node object. But we do call AstNodeCreated custom event on term. - term.OnAstNodeCreated(parseNode); - return; + } else { + //Invoke the default creator compiled when we verified the data + parseNode.AstNode = config.DefaultNodeCreator(); + //Initialize node + var iInit = parseNode.AstNode as IAstNodeInit; + if (iInit != null) + iInit.Init(Context, parseNode); } - //No custom creator. We create node from AstNodeType. We use compiled delegate for this which we create on the fly. - if (term.DefaultAstNodeCreator == null) { - var nodeType = term.AstNodeType ?? Context.Language.Grammar.DefaultNodeType; - term.DefaultAstNodeCreator = CompileDefaultNodeCreator(nodeType); - } - //Invoke the creator - parseNode.AstNode = term.DefaultAstNodeCreator(); - //Initialize node - var iInit = parseNode.AstNode as IAstNodeInit; - if (iInit != null) - iInit.Init(Context, parseNode); //Invoke the event on term term.OnAstNodeCreated(parseNode); }//method @@ -65,36 +114,22 @@ private DefaultAstNodeCreator CompileDefaultNodeCreator(Type nodeType) { return result; } +/* + //A list of of child nodes based on AstPartsMap. By default, the same as ChildNodes + private ParseTreeNodeList _mappedChildNodes; + public ParseTreeNodeList MappedChildNodes { + get { + if (_mappedChildNodes == null) + _mappedChildNodes = GetMappedChildNodes(); + return _mappedChildNodes; + } + } +*/ - }//class - /* OLD code from CoreParser - //Note that we create AST objects for parse nodes only when we pop the node from the stack (when it is a child being added to to its parent). - // So only when we form a parent node, we run thru children in the stack top and check/create their AST nodes. - // This is done to provide correct initialization of List nodes (created with Plus or Star operation). - // We create a parse tree node for a list non-terminal very early, when we encounter its first element. We push the newly created list node into - // the stack. At this moment it is too early to create the AST node for the list. We should wait until all child nodes are parsed and accumulated - // in the stack. Only then, when list construction is finished, we can create AST node and provide it with all list elements. - private void CheckCreateAstNode(ParseTreeNode parseNode) { - try { - //Check preconditions - if (!_grammar.LanguageFlags.IsSet(LanguageFlags.CreateAst)) - return; - if (parseNode.AstNode != null || parseNode.Term.Flags.IsSet(TermFlags.IsTransient) - || parseNode.Term.Flags.IsSet(TermFlags.NoAstNode)) return; - if (Context.Status != ParserStatus.Parsing || Context.HasErrors) return; - //Prepare mapped child node list - CheckCreateMappedChildNodeList(parseNode); - //Actually create node - _grammar.CreateAstNode(Context, parseNode); - if (parseNode.AstNode != null) - parseNode.Term.OnAstNodeCreated(parseNode); - } catch (Exception ex) { - Context.AddParserMessage(ParserErrorLevel.Error, parseNode.Span.Location, Resources.ErrFailedCreateNode, parseNode.Term.Name, ex.Message); - } - } - - */ + + }//class + } diff --git a/Irony/Ast/AstContext.cs b/Irony/Ast/AstContext.cs index aac825a..cc73536 100644 --- a/Irony/Ast/AstContext.cs +++ b/Irony/Ast/AstContext.cs @@ -1,4 +1,16 @@ -using System; +#region License +/* ********************************************************************************** + * Copyright (c) Roman Ivantsov + * This source code is subject to terms and conditions of the MIT License + * for Irony. A copy of the license can be found in the License.txt file + * at the root of this distribution. + * By using this source code in any fashion, you are agreeing to be bound by the terms of the + * MIT License. + * You must not remove this notice from this software. + * **********************************************************************************/ +#endregion + +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -7,8 +19,12 @@ namespace Irony.Ast { public class AstContext { public readonly LanguageData Language; + public Type DefaultNodeType; + public Type DefaultLiteralNodeType; //default node type for literals + public Type DefaultIdentifierNodeType; //default node type for identifiers + public Dictionary Values = new Dictionary(); - public LogMessageList Messages; + public LogMessageList Messages; public AstContext(LanguageData language) { Language = language; diff --git a/Irony/Ast/AstExtensions.cs b/Irony/Ast/AstExtensions.cs new file mode 100644 index 0000000..3d1d2ec --- /dev/null +++ b/Irony/Ast/AstExtensions.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using Irony.Parsing; + +namespace Irony.Ast { + public static class AstExtensions { + + public static ParseTreeNodeList GetMappedChildNodes(this ParseTreeNode node) { + var term = node.Term; + if (!term.HasAstConfig()) + return node.ChildNodes; + var map = term.AstConfig.PartsMap; + //If no map then mapped list is the same as original + if (map == null) return node.ChildNodes; + //Create mapped list + var result = new ParseTreeNodeList(); + for (int i = 0; i < map.Length; i++) { + var index = map[i]; + result.Add(node.ChildNodes[index]); + } + return result; + } + + + } +} diff --git a/Irony/Ast/AstInterfaces.cs b/Irony/Ast/AstInterfaces.cs index d926c33..8a8ed6a 100644 --- a/Irony/Ast/AstInterfaces.cs +++ b/Irony/Ast/AstInterfaces.cs @@ -1,4 +1,16 @@ -using System; +#region License +/* ********************************************************************************** + * Copyright (c) Roman Ivantsov + * This source code is subject to terms and conditions of the MIT License + * for Irony. A copy of the license can be found in the License.txt file + * at the root of this distribution. + * By using this source code in any fashion, you are agreeing to be bound by the terms of the + * MIT License. + * You must not remove this notice from this software. + * **********************************************************************************/ +#endregion + +using System; using System.Collections; using System.Linq; using System.Text; @@ -28,8 +40,5 @@ public interface IAstNodeInit { } - public delegate void AstNodeCreator(AstContext context, ParseTreeNode parseNode); - public delegate object DefaultAstNodeCreator(); - } diff --git a/Irony/Ast/AstNodeConfig.cs b/Irony/Ast/AstNodeConfig.cs index 7ce7fbd..e0c3713 100644 --- a/Irony/Ast/AstNodeConfig.cs +++ b/Irony/Ast/AstNodeConfig.cs @@ -1,52 +1,59 @@ -using System; +#region License +/* ********************************************************************************** + * Copyright (c) Roman Ivantsov + * This source code is subject to terms and conditions of the MIT License + * for Irony. A copy of the license can be found in the License.txt file + * at the root of this distribution. + * By using this source code in any fashion, you are agreeing to be bound by the terms of the + * MIT License. + * You must not remove this notice from this software. + * **********************************************************************************/ +#endregion + +using System; using System.Collections.Generic; using System.Linq; using System.Text; + using Irony.Parsing; namespace Irony.Ast { - public delegate TNode AstNodeCreatorMethod(TContext context, ParseTreeNode parseNode, TNode[] childNodes); - public delegate void AstNodeCreatedHandler(TContext context, ParseTreeNode parseNode, TNode node); + public class AstNodeEventArgs : EventArgs { + public AstNodeEventArgs(ParseTreeNode parseTreeNode) { + ParseTreeNode = parseTreeNode; + } + public readonly ParseTreeNode ParseTreeNode; + public object AstNode { + get { return ParseTreeNode.AstNode; } + } + } + + public delegate void AstNodeCreator(AstContext context, ParseTreeNode parseNode); + public delegate object DefaultAstNodeCreator(); public class AstNodeConfig { + public Type NodeType; + public object Data; //config data passed to AstNode + public AstNodeCreator NodeCreator; // a custom method for creating AST nodes + public DefaultAstNodeCreator DefaultNodeCreator; //default method for creating AST nodes; compiled dynamic method, wrapper around "new nodeType();" + // An optional map (selector, filter) of child AST nodes. This facility provides a way to adjust the "map" of child nodes in various languages to // the structure of a standard AST nodes (that can be shared betweeen languages). // ParseTreeNode object has two properties containing list nodes: ChildNodes and MappedChildNodes. - // If term.Ast.PartsMap is null, these two child node lists are identical and contain all child nodes. + // If term.AstPartsMap is null, these two child node lists are identical and contain all child nodes. // If AstParts is not null, then MappedChildNodes will contain child nodes identified by indexes in the map. // For example, if we set - // term.AstConfig.ChildMap = new int[] {1, 4, 2}; + // term.AstPartsMap = new int[] {1, 4, 2}; // then MappedChildNodes will contain 3 child nodes, which are under indexes 1, 4, 2 in ChildNodes list. // The mapping is performed in CoreParser.cs, method CheckCreateMappedChildNodeList. - public int[] ChildMap; + public int[] PartsMap; - public object Data; // any custom data - // We store untyped refs for creator and createdhandler methods, but provide strongly-typed get/set methods. - // This way we can use AstNodeConfig class in Grammar class without knowing in advance the AST node base types. - // At the time language creator writes the grammar, he would know the AST node types and so he can use strongly-typed - // methods to attach his custom handlers. - private object _creator; // a custom method for creating AST nodes - private object _created; //a handler invoked after AST node is created. - - public void AttachCreator(AstNodeCreatorMethod method) { - _creator = method; - } - - public void AttachNodeCreatedHandler(AstNodeCreatedHandler method) { - _created = method; - } - - public AstNodeCreatorMethod GetCreator() { - if (_creator == null) return null; - return (AstNodeCreatorMethod) _creator; - } - public AstNodeCreatedHandler GetCreatedHandler() { - if (_created == null) return null; - return (AstNodeCreatedHandler)_created; + public bool CanCreateNode() { + return NodeCreator != null || NodeType != null; } - + }//AstNodeConfig class } diff --git a/Irony/Parsing/ConflictResolution/ConditionalParserAction.cs b/Irony/Parsing/ConflictResolution/ConditionalParserAction.cs index af2d532..3ff1b4e 100644 --- a/Irony/Parsing/ConflictResolution/ConditionalParserAction.cs +++ b/Irony/Parsing/ConflictResolution/ConditionalParserAction.cs @@ -63,7 +63,7 @@ public override void Execute(ParsingContext context) { if (DefaultAction == null) { context.AddParserError("Fatal parser error: no conditions matched in conditional parser action, and default action is null." + " State: {0}", context.CurrentParserState.Name); - context.Parser.CoreParser.Recover(); + context.Parser.RecoverFromError(); return; } if (traceEnabled) context.AddTrace(" All conditions failed, executing default action: " + DefaultAction.ToString()); diff --git a/Irony/Parsing/ConflictResolution/CustomActionHintAction.cs b/Irony/Parsing/ConflictResolution/CustomActionHintAction.cs index c8a1b5a..15003b8 100644 --- a/Irony/Parsing/ConflictResolution/CustomActionHintAction.cs +++ b/Irony/Parsing/ConflictResolution/CustomActionHintAction.cs @@ -74,10 +74,20 @@ public CustomParserAction(LanguageData language, ParserState state, } public override void Execute(ParsingContext context) { - //States with DefaultAction do NOT read input + if (context.TracingEnabled) + context.AddTrace(Resources.MsgTraceExecCustomAction); + //States with DefaultAction do NOT read input, so we read it here if (context.CurrentParserInput == null) - context.Parser.CoreParser.ReadInput(); + context.Parser.ReadInput(); + // Remember old state and input; if they don't change after custom action - it is error, we may fall into an endless loop + var oldState = context.CurrentParserState; + var oldInput = context.CurrentParserInput; ExecuteRef(context, this); + //Prevent from falling into an infinite loop + if (context.CurrentParserState == oldState && context.CurrentParserInput == oldInput) { + context.AddParserError(Resources.MsgErrorCustomActionDidNotAdvance); + context.Parser.RecoverFromError(); + } }//method public override string ToString() { diff --git a/Irony/Parsing/Data/Construction/GrammarDataBuilder.cs b/Irony/Parsing/Data/Construction/GrammarDataBuilder.cs index a357730..85b77fa 100644 --- a/Irony/Parsing/Data/Construction/GrammarDataBuilder.cs +++ b/Irony/Parsing/Data/Construction/GrammarDataBuilder.cs @@ -113,9 +113,10 @@ private void FillOperatorReportGroup() { private void InitTermLists() { //Collect terminals and NonTerminals + var empty = _grammar.Empty; foreach (BnfTerm term in _grammarData.AllTerms) { //remember - we may have hints, so it's not only terminals and non-terminals if (term is NonTerminal) _grammarData.NonTerminals.Add((NonTerminal)term); - if (term is Terminal) _grammarData.Terminals.Add((Terminal)term); + if (term is Terminal && term != empty) _grammarData.Terminals.Add((Terminal)term); } //Mark keywords - any "word" symbol directly mentioned in the grammar foreach (var term in _grammarData.Terminals) { @@ -239,12 +240,8 @@ private static void ComputeTailsNullability(GrammarData data) { #region Grammar Validation private void ValidateGrammar() { var createAst = _grammar.LanguageFlags.IsSet(LanguageFlags.CreateAst); - var missingAstTypeSet = new NonTerminalSet(); var invalidTransSet = new NonTerminalSet(); foreach(var nt in _grammarData.NonTerminals) { - //Check that if CreateAst flag is set then AstNodeType or AstNodeCreator is assigned on all non-transient nodes. - if(createAst && nt.AstNodeCreator == null && nt.AstNodeType == null && !nt.Flags.IsSet(TermFlags.NoAstNode)) - missingAstTypeSet.Add(nt); if(nt.Flags.IsSet(TermFlags.IsTransient)) { //List non-terminals cannot be marked transient - otherwise there may be some ambiguities and inconsistencies if (nt.Flags.IsSet(TermFlags.IsList)) @@ -263,8 +260,6 @@ private void ValidateGrammar() { }//foreach prod }//foreac nt - if (missingAstTypeSet.Count > 0) - _language.Errors.Add(GrammarErrorLevel.Warning, null, Resources.ErrNodeTypeNotSetOn, missingAstTypeSet.ToString()); if (invalidTransSet.Count > 0) _language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrTransientNtMustHaveOneTerm,invalidTransSet.ToString()); }//method diff --git a/Irony/Parsing/Data/LanguageData.cs b/Irony/Parsing/Data/LanguageData.cs index 1803e86..4b23d9c 100644 --- a/Irony/Parsing/Data/LanguageData.cs +++ b/Irony/Parsing/Data/LanguageData.cs @@ -25,6 +25,7 @@ public class LanguageData { public readonly GrammarErrorList Errors = new GrammarErrorList(); public GrammarErrorLevel ErrorLevel = GrammarErrorLevel.NoError; public long ConstructionTime; + public bool AstDataVerified; public LanguageData(Grammar grammar) { Grammar = grammar; diff --git a/Irony/Parsing/Grammar/BnfTerm.cs b/Irony/Parsing/Grammar/BnfTerm.cs index 54bf2bd..9deee36 100644 --- a/Irony/Parsing/Grammar/BnfTerm.cs +++ b/Irony/Parsing/Grammar/BnfTerm.cs @@ -15,6 +15,7 @@ using System.Collections.Generic; using System.Text; using System.Reflection; + using Irony.Ast; namespace Irony.Parsing { @@ -63,10 +64,10 @@ public abstract class BnfTerm { #region consructors public BnfTerm(string name) : this(name, name) { } public BnfTerm(string name, string errorAlias, Type nodeType) : this(name, errorAlias) { - AstNodeType = nodeType; + AstConfig.NodeType = nodeType; } public BnfTerm(string name, string errorAlias, AstNodeCreator nodeCreator) : this(name, errorAlias) { - AstNodeCreator = nodeCreator; + AstConfig.NodeCreator = nodeCreator; } public BnfTerm(string name, string errorAlias) { Name = name; @@ -126,49 +127,38 @@ public void SetFlag(TermFlags flag, bool value) { #endregion - #region event: Shifting + #region events: Shifting public event EventHandler Shifting; + public event EventHandler AstNodeCreated; //an event fired after AST node is created. + protected internal void OnShifting(ParsingEventArgs args) { if (Shifting != null) Shifting(this, args); } + + protected internal void OnAstNodeCreated(ParseTreeNode parseNode) { + if (this.AstNodeCreated == null || parseNode.AstNode == null) return; + AstNodeEventArgs args = new AstNodeEventArgs(parseNode); + AstNodeCreated(this, args); + } + #endregion #region AST node creations: AstNodeType, AstNodeCreator, AstNodeCreated - - /* -- new stuff - public Ast.AstNodeConfig AstConfig { + //We autocreate AST config on first GET; + public AstNodeConfig AstConfig { get { if (_astConfig == null) _astConfig = new Ast.AstNodeConfig(); return _astConfig; } set {_astConfig = value; } - } Ast.AstNodeConfig _astConfig; - */ - - public Type AstNodeType; - public object AstData; //config data passed to AstNode - public AstNodeCreator AstNodeCreator; // a custom method for creating AST nodes - public DefaultAstNodeCreator DefaultAstNodeCreator; //default method for creating AST nodes; compiled dynamic method, wrapper around "new nodeType();" - public event EventHandler AstNodeCreated; //an event signalling that AST node is created. - - // An optional map (selector, filter) of child AST nodes. This facility provides a way to adjust the "map" of child nodes in various languages to - // the structure of a standard AST nodes (that can be shared betweeen languages). - // ParseTreeNode object has two properties containing list nodes: ChildNodes and MappedChildNodes. - // If term.AstPartsMap is null, these two child node lists are identical and contain all child nodes. - // If AstParts is not null, then MappedChildNodes will contain child nodes identified by indexes in the map. - // For example, if we set - // term.AstPartsMap = new int[] {1, 4, 2}; - // then MappedChildNodes will contain 3 child nodes, which are under indexes 1, 4, 2 in ChildNodes list. - // The mapping is performed in CoreParser.cs, method CheckCreateMappedChildNodeList. - public int[] AstPartsMap; + } AstNodeConfig _astConfig; - protected internal void OnAstNodeCreated(ParseTreeNode parseNode) { - if (this.AstNodeCreated == null || parseNode.AstNode == null) return; - AstNodeEventArgs args = new AstNodeEventArgs(parseNode); - AstNodeCreated(this, args); + public bool HasAstConfig() { + return _astConfig != null; } + #endregion @@ -239,16 +229,6 @@ internal static BnfExpression Op_Pipe(BnfTerm term1, BnfTerm term2) { public class BnfTermList : List { } public class BnfTermSet : HashSet { } - public class AstNodeEventArgs : EventArgs { - public AstNodeEventArgs(ParseTreeNode parseTreeNode) { - ParseTreeNode = parseTreeNode; - } - public readonly ParseTreeNode ParseTreeNode; - public object AstNode { - get { return ParseTreeNode.AstNode; } - } - } - }//namespace diff --git a/Irony/Parsing/Grammar/Grammar.cs b/Irony/Parsing/Grammar/Grammar.cs index e9277fe..2d62c03 100644 --- a/Irony/Parsing/Grammar/Grammar.cs +++ b/Irony/Parsing/Grammar/Grammar.cs @@ -47,10 +47,6 @@ public class Grammar { // Tokens produced by these terminals will be ignored by parser input. public readonly TerminalSet NonGrammarTerminals = new TerminalSet(); - public Type DefaultNodeType; - public Type DefaultLiteralNodeType; //default node type for literals - public Type DefaultIdentifierNodeType; //default node type for identifiers - /// /// The main root entry for the grammar. /// diff --git a/Irony/Parsing/Parser/CoreParser.cs b/Irony/Parsing/Parser/CoreParser.cs deleted file mode 100644 index 4436938..0000000 --- a/Irony/Parsing/Parser/CoreParser.cs +++ /dev/null @@ -1,195 +0,0 @@ -#region License -/* ********************************************************************************** - * Copyright (c) Roman Ivantsov - * This source code is subject to terms and conditions of the MIT License - * for Irony. A copy of the license can be found in the License.txt file - * at the root of this distribution. - * By using this source code in any fashion, you are agreeing to be bound by the terms of the - * MIT License. - * You must not remove this notice from this software. - * **********************************************************************************/ -#endregion - -using System; -using System.Collections.Generic; -using System.Text; -using System.Collections; -using System.Diagnostics; - - -namespace Irony.Parsing { - // CoreParser class implements LALR parser automaton. Its behavior is controlled by the state transition graph - // with root in Data.InitialState. Each state contains a dictionary of parser actions indexed by input - // element (terminal or non-terminal). - public partial class CoreParser { - - #region Constructors - public CoreParser(Parser parser) { - Parser = parser; - Data = parser.Language.ParserData; - _grammar = Data.Language.Grammar; - } - #endregion - - #region Properties and fields: Parser, Data, _grammar - public readonly Parser Parser; - public readonly ParserData Data; - Grammar _grammar; - - private ParsingContext Context { - get { return Parser.Context; } - } - #endregion - - internal void Reset() { - } - - #region Parse method - public void Parse() { - //main loop - Context.Status = ParserStatus.Parsing; - while(Context.Status == ParserStatus.Parsing) { - ExecuteAction(); - } - }//Parse - - #endregion - - #region reading input - public void ReadInput() { - Token token; - Terminal term; - //Get token from scanner while skipping all comment tokens (but accumulating them in comment block) - do { - token = Parser.Scanner.GetToken(); - term = token.Terminal; - if (term.Category == TokenCategory.Comment) - Context.CurrentCommentTokens.Add(token); - } while (term.Flags.IsSet(TermFlags.IsNonGrammar) && term != _grammar.Eof); - //Check brace token - if (term.Flags.IsSet(TermFlags.IsBrace) && !CheckBraceToken(token)) - token = new Token(_grammar.SyntaxError, token.Location, token.Text, - string.Format(Resources.ErrUnmatchedCloseBrace, token.Text)); - //Create parser input node - Context.CurrentParserInput = new ParseTreeNode(token); - //attach comments if any accumulated to content token - if (token.Terminal.Category == TokenCategory.Content) { - Context.CurrentParserInput.Comments = Context.CurrentCommentTokens; - Context.CurrentCommentTokens = new TokenList(); - } - //Fire event on Terminal - token.Terminal.OnParserInputPreview(Context); - } - - // We assume here that the token is a brace (opening or closing) - private bool CheckBraceToken(Token token) { - if (token.Terminal.Flags.IsSet(TermFlags.IsOpenBrace)) { - Context.OpenBraces.Push(token); - return true; - } - //it is closing brace; check if we have opening brace in the stack - var braces = Context.OpenBraces; - var match = (braces.Count > 0 && braces.Peek().Terminal.IsPairFor == token.Terminal); - if (!match) return false; - //Link both tokens, pop the stack and return true - var openingBrace = braces.Pop(); - openingBrace.OtherBrace = token; - token.OtherBrace = openingBrace; - return true; - } - - #endregion - - #region execute actions - private void ExecuteAction() { - //Read input only if DefaultReduceAction is null - in this case the state does not contain ExpectedSet, - // so parser cannot assist scanner when it needs to select terminal and therefore can fail - if (Context.CurrentParserInput == null && Context.CurrentParserState.DefaultAction == null) - ReadInput(); - //Check scanner error - if (Context.CurrentParserInput != null && Context.CurrentParserInput.IsError) { - Recover(); - return; - } - //Try getting action - var action = GetCurrentAction(); - if (action == null) { - if (CheckPartialInputCompleted()) return; - Recover(); - return; - } - //We have action. Write trace and execute it - if (Context.TracingEnabled) - Context.AddTrace(action.ToString()); - action.Execute(Context); - } - - public void Recover() { - this.Data.ErrorAction.Execute(Context); - } - - private bool CheckPartialInputCompleted() { - bool partialCompleted = (Context.Mode == ParseMode.CommandLine && Context.CurrentParserInput.Term == _grammar.Eof); - if (!partialCompleted) return false; - Context.Status = ParserStatus.AcceptedPartial; - // clean up EOF in input so we can continue parsing next line - Context.CurrentParserInput = null; - return true; - } - - - public ParserAction GetCurrentAction() { - var currState = Context.CurrentParserState; - var currInput = Context.CurrentParserInput; - - if (currState.DefaultAction != null) - return currState.DefaultAction; - ParserAction action; - //First try as keyterm/key symbol; for example if token text = "while", then first try it as a keyword "while"; - // if this does not work, try as an identifier that happens to match a keyword but is in fact identifier - Token inputToken = currInput.Token; - if (inputToken != null && inputToken.KeyTerm != null) { - var keyTerm = inputToken.KeyTerm; - if (currState.Actions.TryGetValue(keyTerm, out action)) { - #region comments - // Ok, we found match as a key term (keyword or special symbol) - // Backpatch the token's term. For example in most cases keywords would be recognized as Identifiers by Scanner. - // Identifier would also check with SymbolTerms table and set AsSymbol field to SymbolTerminal if there exist - // one for token content. So we first find action by Symbol if there is one; if we find action, then we - // patch token's main terminal to AsSymbol value. This is important for recognizing keywords (for colorizing), - // and for operator precedence algorithm to work when grammar uses operators like "AND", "OR", etc. - //TODO: This might be not quite correct action, and we can run into trouble with some languages that have keywords that - // are not reserved words. But proper implementation would require substantial addition to parser code: - // when running into errors, we need to check the stack for places where we made this "interpret as Symbol" - // decision, roll back the stack and try to reinterpret as identifier - #endregion - inputToken.SetTerminal(keyTerm); - currInput.Term = keyTerm; - currInput.Precedence = keyTerm.Precedence; - currInput.Associativity = keyTerm.Associativity; - return action; - } - } - //Try to get by main Terminal, only if it is not the same as symbol - if (currState.Actions.TryGetValue(currInput.Term, out action)) - return action; - //If input is EOF and NewLineBeforeEof flag is set, try using NewLine to find action - if (currInput.Term == _grammar.Eof && _grammar.LanguageFlags.IsSet(LanguageFlags.NewLineBeforeEOF) && - currState.Actions.TryGetValue(_grammar.NewLine, out action)) { - //There's no action for EOF but there's action for NewLine. Let's add newLine token as input, just in case - // action code wants to check input - it should see NewLine. - var newLineToken = new Token(_grammar.NewLine, currInput.Token.Location, "\r\n", null); - var newLineNode = new ParseTreeNode(newLineToken); - Context.CurrentParserInput = newLineNode; - return action; - }//if - return null; - } - - #endregion - - }//class - - - -}//namespace diff --git a/Irony/Parsing/Parser/ParseTree.cs b/Irony/Parsing/Parser/ParseTree.cs index a331214..8f62058 100644 --- a/Irony/Parsing/Parser/ParseTree.cs +++ b/Irony/Parsing/Parser/ParseTree.cs @@ -70,29 +70,6 @@ public override string ToString() { return Term.GetParseNodeCaption(this); }//method - //A list of of child nodes based on AstPartsMap. By default, the same as ChildNodes - private ParseTreeNodeList _mappedChildNodes; - public ParseTreeNodeList MappedChildNodes { - get { - if (_mappedChildNodes == null) - _mappedChildNodes = GetMappedChildNodes(); - return _mappedChildNodes; - } - } - private ParseTreeNodeList GetMappedChildNodes() { - var map = Term.AstPartsMap; - //If no map then mapped list is the same as original - if (map == null) - return ChildNodes; - //Create mapped list - var nodes = new ParseTreeNodeList(); - for (int i = 0; i < map.Length; i++) { - var index = map[i]; - nodes.Add(ChildNodes[index]); - } - return nodes; - } - public string FindTokenAndGetText() { var tkn = FindToken(); @@ -103,18 +80,12 @@ public Token FindToken() { } private static Token FindFirstChildTokenRec(ParseTreeNode node) { if (node.Token != null) return node.Token; - foreach (var child in node.MappedChildNodes) { + foreach (var child in node.ChildNodes) { var tkn = FindFirstChildTokenRec(child); if (tkn != null) return tkn; } return null; } - public ParseTreeNode FirstChild { - get { return MappedChildNodes[0]; } - } - public ParseTreeNode LastChild { - get { return MappedChildNodes[MappedChildNodes.Count - 1]; } - } /// Returns true if the node is punctuation or it is transient with empty child list. /// True if parser can safely ignore this node. diff --git a/Irony/Parsing/Parser/ParseTreeExtensions.cs b/Irony/Parsing/Parser/ParseTreeExtensions.cs index be6529b..a9e7042 100644 --- a/Irony/Parsing/Parser/ParseTreeExtensions.cs +++ b/Irony/Parsing/Parser/ParseTreeExtensions.cs @@ -45,8 +45,9 @@ public static XmlDocument ToXmlDocument(this ParseTree parseTree) { public static XmlElement ToXmlElement(this ParseTreeNode node, XmlDocument ownerDocument) { var xElem = ownerDocument.CreateElement("Node"); xElem.SetAttribute("Term", node.Term.Name); - if (node.Term.AstNodeType != null) - xElem.SetAttribute("AstNodeType", node.Term.AstNodeType.Name); + var term = node.Term; + if (term.HasAstConfig() && term.AstConfig.NodeType != null) + xElem.SetAttribute("AstNodeType", term.AstConfig.NodeType.Name); if (node.Token != null) { xElem.SetAttribute("Terminal", node.Term.GetType().Name); //xElem.SetAttribute("Text", node.Token.Text); diff --git a/Irony/Parsing/Parser/Parser.cs b/Irony/Parsing/Parser/Parser.cs index bf3b643..ded91c7 100644 --- a/Irony/Parsing/Parser/Parser.cs +++ b/Irony/Parsing/Parser/Parser.cs @@ -20,8 +20,10 @@ namespace Irony.Parsing { //Parser class represents combination of scanner and LALR parser (CoreParser) public class Parser { - public readonly LanguageData Language; - public readonly CoreParser CoreParser; + public readonly LanguageData Language; + public readonly ParserData Data; + private Grammar _grammar; + //public readonly CoreParser CoreParser; public readonly Scanner Scanner; public ParsingContext Context { get; internal set; } public readonly NonTerminal Root; @@ -32,9 +34,10 @@ public Parser(Grammar grammar) : this (new LanguageData(grammar)) { } public Parser(LanguageData language) : this(language, null) {} public Parser(LanguageData language, NonTerminal root) { Language = language; - Context = new ParsingContext(this); + Data = Language.ParserData; + _grammar = Language.Grammar; + Context = new ParsingContext(this); Scanner = new Scanner(this); - CoreParser = new CoreParser(this); Root = root; if(Root == null) { Root = Language.Grammar.Root; @@ -48,7 +51,6 @@ public Parser(LanguageData language, NonTerminal root) { internal void Reset() { Context.Reset(); - CoreParser.Reset(); Scanner.Reset(); } @@ -70,7 +72,7 @@ public ParseTree Parse(string sourceText, string fileName) { Context.Status = ParserStatus.Parsing; var sw = new Stopwatch(); sw.Start(); - CoreParser.Parse(); + ParseAll(); //Set Parse status var parseTree = Context.CurrentParseTree; bool hasErrors = parseTree.HasErrors(); @@ -81,7 +83,7 @@ public ParseTree Parse(string sourceText, string fileName) { else parseTree.Status = ParseTreeStatus.Parsed; //Build AST if no errors and AST flag is set - bool createAst = Language.Grammar.LanguageFlags.IsSet(LanguageFlags.CreateAst); + bool createAst = _grammar.LanguageFlags.IsSet(LanguageFlags.CreateAst); if (createAst && !hasErrors) Language.Grammar.BuildAst(Language, parseTree); //Done; record the time @@ -92,6 +94,14 @@ public ParseTree Parse(string sourceText, string fileName) { return parseTree; } + private void ParseAll() { + //main loop + Context.Status = ParserStatus.Parsing; + while (Context.Status == ParserStatus.Parsing) { + ExecuteNextAction(); + } + }//ParseAll method + public ParseTree ScanOnly(string sourceText, string fileName) { Context.CurrentParseTree = new ParseTree(sourceText, fileName); Context.Source = new SourceStream(sourceText, Language.Grammar.CaseSensitive, Context.TabWidth); @@ -102,6 +112,143 @@ public ParseTree ScanOnly(string sourceText, string fileName) { return Context.CurrentParseTree; } + #region Parser Action execution + private void ExecuteNextAction() { + //Read input only if DefaultReduceAction is null - in this case the state does not contain ExpectedSet, + // so parser cannot assist scanner when it needs to select terminal and therefore can fail + if (Context.CurrentParserInput == null && Context.CurrentParserState.DefaultAction == null) + ReadInput(); + //Check scanner error + if (Context.CurrentParserInput != null && Context.CurrentParserInput.IsError) { + RecoverFromError(); + return; + } + //Try getting action + var action = GetNextAction(); + if (action == null) { + if (CheckPartialInputCompleted()) return; + RecoverFromError(); + return; + } + //We have action. Write trace and execute it + if (Context.TracingEnabled) + Context.AddTrace(action.ToString()); + action.Execute(Context); + } + + internal ParserAction GetNextAction() { + var currState = Context.CurrentParserState; + var currInput = Context.CurrentParserInput; + + if (currState.DefaultAction != null) + return currState.DefaultAction; + ParserAction action; + //First try as keyterm/key symbol; for example if token text = "while", then first try it as a keyword "while"; + // if this does not work, try as an identifier that happens to match a keyword but is in fact identifier + Token inputToken = currInput.Token; + if (inputToken != null && inputToken.KeyTerm != null) { + var keyTerm = inputToken.KeyTerm; + if (currState.Actions.TryGetValue(keyTerm, out action)) { + #region comments + // Ok, we found match as a key term (keyword or special symbol) + // Backpatch the token's term. For example in most cases keywords would be recognized as Identifiers by Scanner. + // Identifier would also check with SymbolTerms table and set AsSymbol field to SymbolTerminal if there exist + // one for token content. So we first find action by Symbol if there is one; if we find action, then we + // patch token's main terminal to AsSymbol value. This is important for recognizing keywords (for colorizing), + // and for operator precedence algorithm to work when grammar uses operators like "AND", "OR", etc. + //TODO: This might be not quite correct action, and we can run into trouble with some languages that have keywords that + // are not reserved words. But proper implementation would require substantial addition to parser code: + // when running into errors, we need to check the stack for places where we made this "interpret as Symbol" + // decision, roll back the stack and try to reinterpret as identifier + #endregion + inputToken.SetTerminal(keyTerm); + currInput.Term = keyTerm; + currInput.Precedence = keyTerm.Precedence; + currInput.Associativity = keyTerm.Associativity; + return action; + } + } + //Try to get by main Terminal, only if it is not the same as symbol + if (currState.Actions.TryGetValue(currInput.Term, out action)) + return action; + //If input is EOF and NewLineBeforeEof flag is set, try using NewLine to find action + if (currInput.Term == _grammar.Eof && _grammar.LanguageFlags.IsSet(LanguageFlags.NewLineBeforeEOF) && + currState.Actions.TryGetValue(_grammar.NewLine, out action)) { + //There's no action for EOF but there's action for NewLine. Let's add newLine token as input, just in case + // action code wants to check input - it should see NewLine. + var newLineToken = new Token(_grammar.NewLine, currInput.Token.Location, "\r\n", null); + var newLineNode = new ParseTreeNode(newLineToken); + Context.CurrentParserInput = newLineNode; + return action; + }//if + return null; + } + + #endregion + + #region reading input + public void ReadInput() { + Token token; + Terminal term; + //Get token from scanner while skipping all comment tokens (but accumulating them in comment block) + do { + token = Scanner.GetToken(); + term = token.Terminal; + if (term.Category == TokenCategory.Comment) + Context.CurrentCommentTokens.Add(token); + } while (term.Flags.IsSet(TermFlags.IsNonGrammar) && term != _grammar.Eof); + //Check brace token + if (term.Flags.IsSet(TermFlags.IsBrace) && !CheckBraceToken(token)) + token = new Token(_grammar.SyntaxError, token.Location, token.Text, + string.Format(Resources.ErrUnmatchedCloseBrace, token.Text)); + //Create parser input node + Context.CurrentParserInput = new ParseTreeNode(token); + //attach comments if any accumulated to content token + if (token.Terminal.Category == TokenCategory.Content) { + Context.CurrentParserInput.Comments = Context.CurrentCommentTokens; + Context.CurrentCommentTokens = new TokenList(); + } + //Fire event on Terminal + token.Terminal.OnParserInputPreview(Context); + } + #endregion + + #region Error Recovery + public void RecoverFromError() { + this.Data.ErrorAction.Execute(Context); + } + #endregion + + #region Utilities + private bool CheckPartialInputCompleted() { + bool partialCompleted = (Context.Mode == ParseMode.CommandLine && Context.CurrentParserInput.Term == _grammar.Eof); + if (!partialCompleted) return false; + Context.Status = ParserStatus.AcceptedPartial; + // clean up EOF in input so we can continue parsing next line + Context.CurrentParserInput = null; + return true; + } + + // We assume here that the token is a brace (opening or closing) + private bool CheckBraceToken(Token token) { + if (token.Terminal.Flags.IsSet(TermFlags.IsOpenBrace)) { + Context.OpenBraces.Push(token); + return true; + } + //it is closing brace; check if we have opening brace in the stack + var braces = Context.OpenBraces; + var match = (braces.Count > 0 && braces.Peek().Terminal.IsPairFor == token.Terminal); + if (!match) return false; + //Link both tokens, pop the stack and return true + var openingBrace = braces.Pop(); + openingBrace.OtherBrace = token; + token.OtherBrace = openingBrace; + return true; + } + #endregion + + + }//class }//namespace diff --git a/Irony/Parsing/Parser/ParserActions/ErrorRecoveryParserAction.cs b/Irony/Parsing/Parser/ParserActions/ErrorRecoveryParserAction.cs index 10415b4..76ab491 100644 --- a/Irony/Parsing/Parser/ParserActions/ErrorRecoveryParserAction.cs +++ b/Irony/Parsing/Parser/ParserActions/ErrorRecoveryParserAction.cs @@ -4,16 +4,10 @@ using System.Text; namespace Irony.Parsing { + //TODO: Improve recovery by adding automatic injection of missing tokens. + // Make sure we ALWAYS have output parse tree, even if it is messed up public class ErrorRecoveryParserAction : ParserAction { - /* -- OLD CODE - public void ResetLocationAndClearInput(SourceLocation location, int position) { - context.CurrentParserInput = null; - context.ParserInputStack.Clear(); - context.SetSourceLocation(location); - } - */ - public override void Execute(ParsingContext context) { context.Status = ParserStatus.Error; var grammar = context.Language.Grammar; @@ -36,7 +30,7 @@ public override void Execute(ParsingContext context) { protected bool TryRecoverFromError(ParsingContext context) { var grammar = context.Language.Grammar; - var coreParser = context.Parser.CoreParser; + var parser = context.Parser; //1. We need to find a state in the stack that has a shift item based on error production (with error token), // and error terminal is current. This state would have a shift action on error token. ParserAction errorShiftAction = FindErrorShiftActionInStack(context); @@ -50,20 +44,19 @@ protected bool TryRecoverFromError(ParsingContext context) { context.AddTrace(Resources.MsgTraceRecoverShiftTillEnd); while (true) { if (context.CurrentParserInput == null) - coreParser.ReadInput(); + parser.ReadInput(); if (context.CurrentParserInput.Term == grammar.Eof) return false; //Check if we can reduce - var nextAction = coreParser.GetCurrentAction(); + var nextAction = parser.GetNextAction(); if (nextAction == null) { - coreParser.ReadInput(); + parser.ReadInput(); continue; } if (nextAction is ReduceParserAction) { //We are reducing a fragment containing error - this is the end of recovery //Clear all input token queues and buffered input, reset location back to input position token queues; context.SetSourceLocation(context.CurrentParserInput.Span.Location); - context.CurrentParserInput = null; //Reduce error production - it creates parent non-terminal that "hides" error inside context.AddTrace(Resources.MsgTraceRecoverReducing); @@ -72,6 +65,7 @@ protected bool TryRecoverFromError(ParsingContext context) { return true; //we recovered } // If it is not reduce, simply execute it (it is most likely shift) + context.AddTrace(Resources.MsgTraceRecoverAction, nextAction); nextAction.Execute(context); //shift input token } }//method diff --git a/Irony/Parsing/Scanner/SourceStream.cs b/Irony/Parsing/Scanner/SourceStream.cs index 2e26beb..8e3fd85 100644 --- a/Irony/Parsing/Scanner/SourceStream.cs +++ b/Irony/Parsing/Scanner/SourceStream.cs @@ -132,7 +132,7 @@ public override string ToString() { //Computes the Location info (line, col) for a new source position. private void SetNewPosition(int newPosition) { if (newPosition < Position) - throw new Exception("Cannot move back in the source."); + throw new Exception(Resources.ErrCannotMoveBackInSource); int p = Position; int col = Location.Column; int line = Location.Line; diff --git a/Irony/Parsing/Terminals/ConstantTerminal.cs b/Irony/Parsing/Terminals/ConstantTerminal.cs index b8194b3..f07f8d9 100644 --- a/Irony/Parsing/Terminals/ConstantTerminal.cs +++ b/Irony/Parsing/Terminals/ConstantTerminal.cs @@ -23,7 +23,7 @@ public class ConstantTerminal : Terminal { public readonly ConstantsTable Constants = new ConstantsTable(); public ConstantTerminal(string name, Type nodeType) : base(name) { base.SetFlag(TermFlags.IsConstant); - AstNodeType = nodeType; + base.AstConfig.NodeType = nodeType; } public void Add(string lexeme, object value) { diff --git a/Irony/Parsing/Terminals/IdentifierTerminal.cs b/Irony/Parsing/Terminals/IdentifierTerminal.cs index fd7d60d..f0f6928 100644 --- a/Irony/Parsing/Terminals/IdentifierTerminal.cs +++ b/Irony/Parsing/Terminals/IdentifierTerminal.cs @@ -85,9 +85,6 @@ public void AddPrefix(string prefix, IdOptions options) { #region overrides public override void Init(GrammarData grammarData) { base.Init(grammarData); - if (this.AstNodeType == null && this.AstNodeCreator == null) - this.AstNodeType = grammarData.Grammar.DefaultIdentifierNodeType; - _allCharsSet = new CharHashSet(Grammar.CaseSensitive); _allCharsSet.UnionWith(AllChars.ToCharArray()); diff --git a/Irony/Parsing/Terminals/NumberLiteral.cs b/Irony/Parsing/Terminals/NumberLiteral.cs index ded79d0..3dfa53d 100644 --- a/Irony/Parsing/Terminals/NumberLiteral.cs +++ b/Irony/Parsing/Terminals/NumberLiteral.cs @@ -17,6 +17,7 @@ using System.Globalization; using System.Text; using System.Diagnostics; +using Irony.Ast; namespace Irony.Parsing { using BigInteger = System.Numerics.BigInteger; //Microsoft.Scripting.Math.BigInteger; @@ -65,10 +66,10 @@ public class ExponentsTable : Dictionary { } public NumberLiteral(string name) : this(name, NumberOptions.Default) { } public NumberLiteral(string name, NumberOptions options, Type astNodeType) : this(name, options) { - base.AstNodeType = astNodeType; + base.AstConfig.NodeType = astNodeType; } public NumberLiteral(string name, NumberOptions options, AstNodeCreator astNodeCreator) : this(name, options) { - base.AstNodeCreator = astNodeCreator; + base.AstConfig.NodeCreator = astNodeCreator; } public NumberLiteral(string name, NumberOptions options) : base(name) { Options = options; diff --git a/Irony/Parsing/Terminals/StringLiteral.cs b/Irony/Parsing/Terminals/StringLiteral.cs index a0043d2..4630bef 100644 --- a/Irony/Parsing/Terminals/StringLiteral.cs +++ b/Irony/Parsing/Terminals/StringLiteral.cs @@ -88,11 +88,11 @@ public StringLiteral(string name, string startEndSymbol) : this(name, startEndSy public StringLiteral(string name, string startEndSymbol, StringOptions options, Type astNodeType) : this(name, startEndSymbol, options) { - base.AstNodeType = astNodeType; + base.AstConfig.NodeType = astNodeType; } public StringLiteral(string name, string startEndSymbol, StringOptions options, AstNodeCreator astNodeCreator) : this(name, startEndSymbol, options) { - base.AstNodeCreator = astNodeCreator; + base.AstConfig.NodeCreator = astNodeCreator; } public void AddStartEnd(string startEndSymbol, StringOptions stringOptions) { @@ -115,8 +115,6 @@ public void AddPrefix(string prefix, StringOptions flags) { #region overrides: Init, GetFirsts, ReadBody, etc... public override void Init(GrammarData grammarData) { base.Init(grammarData); - if (AstNodeType == null) - base.AstNodeType = grammarData.Grammar.DefaultLiteralNodeType; _startSymbolsFirsts = string.Empty; if (_subtypes.Count == 0) { grammarData.Language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrInvStrDef, this.Name); //"Error in string literal [{0}]: No start/end symbols specified." @@ -146,7 +144,7 @@ public override void Init(GrammarData grammarData) { //For templates only if(isTemplate) { //Check that template settings object is provided - var templateSettings = this.AstData as StringTemplateSettings; + var templateSettings = this.AstConfig.Data as StringTemplateSettings; if(templateSettings == null) grammarData.Language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrTemplNoSettings, this.Name); //"Error in string literal [{0}]: IsTemplate flag is set, but TemplateSettings is not provided." else if (templateSettings.ExpressionRoot == null) diff --git a/Irony/Parsing/Terminals/_Terminal.cs b/Irony/Parsing/Terminals/_Terminal.cs index 0538429..fd03303 100644 --- a/Irony/Parsing/Terminals/_Terminal.cs +++ b/Irony/Parsing/Terminals/_Terminal.cs @@ -61,11 +61,6 @@ public Terminal(string name, TokenCategory category, TermFlags flags) : base(na #region virtual methods: GetFirsts(), TryMatch, Init, TokenToString public override void Init(GrammarData grammarData) { base.Init(grammarData); - //By default for Literal terminals assign node type in Grammar.DefaultLiteralNodeType - bool assignLiteralType = (AstNodeType == null && AstNodeCreator == null && - Flags.IsSet(TermFlags.IsLiteral) && Grammar.LanguageFlags.IsSet(LanguageFlags.CreateAst)); - if (assignLiteralType) - AstNodeType = this.Grammar.DefaultLiteralNodeType; } //"Firsts" (chars) collections are used for quick search for possible matching terminal(s) using current character in the input stream. diff --git a/Irony/Resources.Designer.cs b/Irony/Resources.Designer.cs index 1c011dc..b7d507c 100644 --- a/Irony/Resources.Designer.cs +++ b/Irony/Resources.Designer.cs @@ -106,7 +106,7 @@ public static string ErrArgListNotFound { } /// - /// Looks up a localized string similar to Invalide operation, attempt to assign constant or literal value.. + /// Looks up a localized string similar to Invalid operation, attempt to assign to a constant or literal value.. /// public static string ErrAssignLiteralValue { get { @@ -168,6 +168,15 @@ public static string ErrCannotConvertValueToType { } } + /// + /// Looks up a localized string similar to Fatal error in parser: attempt to move back in the source.. + /// + public static string ErrCannotMoveBackInSource { + get { + return ResourceManager.GetString("ErrCannotMoveBackInSource", resourceCulture); + } + } + /// /// Looks up a localized string similar to {0} State {1} on inputs: {2}. /// @@ -403,7 +412,7 @@ public static string ErrNoClosingBrace { } /// - /// Looks up a localized string similar to Warning: AstNodeType or AstNodeCreator is not set on non-terminals: {0}.. + /// Looks up a localized string similar to AstNodeType or AstNodeCreator is not set on non-terminals: {0}. Either set Term.AstConfig.NodeType, or provide default values in AstContext.. /// public static string ErrNodeTypeNotSetOn { get { @@ -897,6 +906,15 @@ public static string MsgDefaultConsoleTitle { } } + /// + /// Looks up a localized string similar to Custom action did not execute: parser state or input did not change.. + /// + public static string MsgErrorCustomActionDidNotAdvance { + get { + return ResourceManager.GetString("MsgErrorCustomActionDidNotAdvance", resourceCulture); + } + } + /// /// Looks up a localized string similar to Exit console (y/n)?. /// @@ -942,6 +960,15 @@ public static string MsgTraceConflictResolved { } } + /// + /// Looks up a localized string similar to Executing custom action. + /// + public static string MsgTraceExecCustomAction { + get { + return ResourceManager.GetString("MsgTraceExecCustomAction", resourceCulture); + } + } + /// /// Looks up a localized string similar to Operator - resolved to {0}. /// diff --git a/Irony/Resources.resx b/Irony/Resources.resx index 3eb0078..b33446f 100644 --- a/Irony/Resources.resx +++ b/Irony/Resources.resx @@ -212,7 +212,7 @@ {0} State {1} on inputs: {2} - Warning: AstNodeType or AstNodeCreator is not set on non-terminals: {0}. + AstNodeType or AstNodeCreator is not set on non-terminals: {0}. Either set Term.AstConfig.NodeType, or provide default values in AstContext. ParserDataBuilder error: inadequate state {0}, reduce item '{1}' has no lookaheads. @@ -392,7 +392,7 @@ Invalid arguments for IncDecNode AST node: either first or second argument should be '--' or '++'. - Invalide operation, attempt to assign constant or literal value. + Invalid operation, attempt to assign to a constant or literal value. Error in string literal [{0}]: IsTemplate flag is set, but TemplateSettings is not provided in AstNodeConfig property. @@ -442,4 +442,13 @@ Expected new line symbol. + + Custom action did not execute: parser state or input did not change. + + + Executing custom action + + + Fatal error in parser: attempt to move back in the source. + \ No newline at end of file