finish refactoring of AST build process

Irony.GrammarExplorer/fmGrammarExplorer.cs

@@ -188,10 +188,10 @@ private void ShowParserConstructionResults() {
       txtNonTerms.Text = string.Empty;
       txtParserStates.Text = string.Empty;
       tabBottom.SelectedTab = pageLanguage;
-      if (_parser == null) return;
-      txtTerms.Text = ParserDataPrinter.PrintTerminals(_parser.Language);
-      txtNonTerms.Text = ParserDataPrinter.PrintNonTerminals(_parser.Language);
-      txtParserStates.Text = ParserDataPrinter.PrintStateList(_parser.Language);
+      if (_parser == null) return; 
+      txtTerms.Text = ParserDataPrinter.PrintTerminals(_language);
+      txtNonTerms.Text = ParserDataPrinter.PrintNonTerminals(_language);
+      txtParserStates.Text = ParserDataPrinter.PrintStateList(_language);
       ShowGrammarErrors();
     }//method
 

Irony.Interpreter/Ast/AstContext/InterpreterAstContext.cs

@@ -12,7 +12,10 @@ public class InterpreterAstContext : AstContext {
     public readonly OperatorHandler OperatorHandler; 
 
     public InterpreterAstContext(LanguageData language, OperatorHandler operatorHandler = null) : base(language) {
-      OperatorHandler = operatorHandler ?? new OperatorHandler(language.Grammar.CaseSensitive); 
+      OperatorHandler = operatorHandler ?? new OperatorHandler(language.Grammar.CaseSensitive);
+      base.DefaultIdentifierNodeType = typeof(IdentifierNode);
+      base.DefaultLiteralNodeType = typeof(LiteralValueNode);
+      base.DefaultNodeType = null; 
     }
 
   }//class

Irony.Interpreter/Ast/Expressions/BinaryOperationNode.cs

@@ -33,9 +33,10 @@ public BinaryOperationNode() { }
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      Left = AddChild("Arg", treeNode.MappedChildNodes[0]);
-      Right = AddChild("Arg", treeNode.MappedChildNodes[2]);
-      var opToken = treeNode.MappedChildNodes[1].FindToken();
+      var nodes = treeNode.GetMappedChildNodes();
+      Left = AddChild("Arg", nodes[0]);
+      Right = AddChild("Arg", nodes[2]);
+      var opToken = nodes[1].FindToken();
       OpSymbol = opToken.Text;
       var ictxt = context as InterpreterAstContext;
       Op = ictxt.OperatorHandler.GetOperatorExpressionType(OpSymbol);

Irony.Interpreter/Ast/Expressions/ExpressionListNode.cs

@@ -25,7 +25,7 @@ public class ExpressionListNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      foreach (var child in treeNode.MappedChildNodes) {
+      foreach (var child in treeNode.ChildNodes) {
           AddChild(NodeUseType.Parameter, "expr", child); 
       }
       AsString = "Expression list";

Irony.Interpreter/Ast/Expressions/IfNode.cs

@@ -25,10 +25,11 @@ public class IfNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      Test = AddChild("Test", treeNode.MappedChildNodes[0]);
-      IfTrue = AddChild("IfTrue", treeNode.MappedChildNodes[1]);
-      if (treeNode.MappedChildNodes.Count > 2)
-        IfFalse = AddChild("IfFalse", treeNode.MappedChildNodes[2]);
+      var nodes = treeNode.GetMappedChildNodes();
+      Test = AddChild("Test", nodes[0]);
+      IfTrue = AddChild("IfTrue", nodes[1]);
+      if (nodes.Count > 2)
+        IfFalse = AddChild("IfFalse", nodes[2]);
     }
 
     protected override object DoEvaluate(ScriptThread thread) {

Irony.Interpreter/Ast/Expressions/IncDecNode.cs

@@ -31,21 +31,22 @@ public class IncDecNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      FindOpAndDetectPostfix(treeNode); 
+      var nodes = treeNode.GetMappedChildNodes();
+      FindOpAndDetectPostfix(nodes); 
       int argIndex = IsPostfix? 0 : 1;
-      Argument = AddChild(NodeUseType.ValueReadWrite, "Arg", treeNode.MappedChildNodes[argIndex]);
+      Argument = AddChild(NodeUseType.ValueReadWrite, "Arg", nodes[argIndex]);
       BinaryOpSymbol = OpSymbol[0].ToString(); //take a single char out of ++ or --
       var interpContext = (InterpreterAstContext)context; 
       BinaryOp = interpContext.OperatorHandler.GetOperatorExpressionType(BinaryOpSymbol); 
       base.AsString = OpSymbol + (IsPostfix ? "(postfix)" : "(prefix)");
     }
 
-    private void FindOpAndDetectPostfix(ParseTreeNode treeNode) {
+    private void FindOpAndDetectPostfix(ParseTreeNodeList mappedNodes) {
       IsPostfix = false; //assume it 
-      OpSymbol = treeNode.MappedChildNodes[0].FindTokenAndGetText();
+      OpSymbol = mappedNodes[0].FindTokenAndGetText();
       if (OpSymbol == "--" || OpSymbol == "++") return;
       IsPostfix = true;
-      OpSymbol = treeNode.MappedChildNodes[1].FindTokenAndGetText();
+      OpSymbol = mappedNodes[1].FindTokenAndGetText();
     }
 
     protected override object DoEvaluate(ScriptThread thread) {

Irony.Interpreter/Ast/Expressions/IndexedAccessNode.cs

@@ -15,8 +15,9 @@ public class IndexedAccessNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      _target = AddChild("Target", treeNode.FirstChild);
-      _index = AddChild("Index", treeNode.LastChild); 
+      var nodes = treeNode.GetMappedChildNodes();
+      _target = AddChild("Target", nodes.First());
+      _index = AddChild("Index", nodes.Last()); 
       AsString = "[" + _index + "]";
     }
 

Irony.Interpreter/Ast/Expressions/MemberAccessNode.cs

@@ -17,8 +17,9 @@ public class MemberAccessNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      _left = AddChild("Target", treeNode.FirstChild);
-      var right = treeNode.LastChild;
+      var nodes = treeNode.GetMappedChildNodes();
+      _left = AddChild("Target", nodes[0]);
+      var right = nodes[nodes.Count - 1];
       _memberName = right.FindTokenAndGetText();
       ErrorAnchor = right.Span.Location; 
       AsString = "." + _memberName;

Irony.Interpreter/Ast/Expressions/UnaryOperationNode.cs

@@ -27,8 +27,9 @@ public class UnaryOperationNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      OpSymbol = treeNode.MappedChildNodes[0].FindTokenAndGetText();
-      Argument = AddChild("Arg", treeNode.MappedChildNodes[1]);
+      var nodes = treeNode.GetMappedChildNodes();
+      OpSymbol = nodes[0].FindTokenAndGetText();
+      Argument = AddChild("Arg", nodes[1]);
       base.AsString = OpSymbol + "(unary op)";
       var interpContext = (InterpreterAstContext)context; 
       base.ExpressionType = interpContext.OperatorHandler.GetUnaryOperatorExpressionType(OpSymbol);

Irony.Interpreter/Ast/Functions/FunctionCallNode.cs

@@ -30,10 +30,11 @@ public class FunctionCallNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      TargetRef = AddChild("Target", treeNode.MappedChildNodes[0]);
+      var nodes = treeNode.GetMappedChildNodes();
+      TargetRef = AddChild("Target", nodes[0]);
       TargetRef.UseType = NodeUseType.CallTarget;
-      _targetName = treeNode.MappedChildNodes[0].FindTokenAndGetText();
-      Arguments = AddChild("Args", treeNode.MappedChildNodes[1]);
+      _targetName = nodes[0].FindTokenAndGetText();
+      Arguments = AddChild("Args", nodes[1]);
       AsString = "Call " + _targetName;
     }
 

Irony.Interpreter/Ast/Functions/FunctionDefNode.cs

@@ -30,9 +30,10 @@ public class FunctionDefNode : AstNode {
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
       //child #0 is usually a keyword like "def"
-      NameNode = AddChild("Name", treeNode.MappedChildNodes[1]);
-      Parameters = AddChild("Parameters", treeNode.MappedChildNodes[2]);
-      Body = AddChild("Body", treeNode.MappedChildNodes[3]);
+      var nodes = treeNode.GetMappedChildNodes();
+      NameNode = AddChild("Name", nodes[1]);
+      Parameters = AddChild("Parameters", nodes[2]);
+      Body = AddChild("Body", nodes[3]);
       AsString = "<Function " + NameNode.AsString + ">";
       Body.SetIsTail(); //this will be propagated to the last statement
     }

Irony.Interpreter/Ast/Functions/ParamListNode.cs

@@ -24,7 +24,7 @@ public class ParamListNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      foreach (var child in treeNode.MappedChildNodes)
+      foreach (var child in treeNode.ChildNodes)
         AddChild(NodeUseType.Parameter, "param", child); 
       AsString = "param_list[" + ChildNodes.Count + "]";
     }

Irony.Interpreter/Ast/PrimitiveNodes/StringTemplateNode.cs

@@ -68,7 +68,7 @@ public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
       _template = treeNode.Token.ValueString;
       _tokenText = treeNode.Token.Text;
-      _templateSettings = treeNode.Term.AstData as StringTemplateSettings; 
+      _templateSettings = treeNode.Term.AstConfig.Data as StringTemplateSettings; 
       ParseSegments(context); 
       AsString = "\"" + _template + "\" (templated string)"; 
     }

Irony.Interpreter/Ast/Statements/AssignmentNode.cs

@@ -30,14 +30,15 @@ public class AssignmentNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      Target = AddChild(NodeUseType.ValueWrite, "To", treeNode.MappedChildNodes[0]);
+      var nodes = treeNode.GetMappedChildNodes();
+      Target = AddChild(NodeUseType.ValueWrite, "To", nodes[0]);
       //Get Op and baseOp if it is combined assignment
-      AssignmentOp = treeNode.MappedChildNodes[1].FindTokenAndGetText();
+      AssignmentOp = nodes[1].FindTokenAndGetText();
       if (string.IsNullOrEmpty(AssignmentOp))
         AssignmentOp = "=";
       BinaryExpressionType = CustomExpressionTypes.NotAnExpression;
       //There maybe an "=" sign in the middle, or not - if it is marked as punctuation; so we just take the last node in child list
-      Expression = AddChild(NodeUseType.ValueRead, "Expr", treeNode.LastChild);
+      Expression = AddChild(NodeUseType.ValueRead, "Expr", nodes[nodes.Count - 1]);
       AsString = AssignmentOp + " (assignment)";
       // TODO: this is not always correct: in Pascal the assignment operator is :=.
       IsAugmented = AssignmentOp.Length > 1;

Irony.Interpreter/Ast/Statements/StatementListNode.cs

@@ -25,7 +25,8 @@ public class StatementListNode : AstNode {
 
     public override void Init(AstContext context, ParseTreeNode treeNode) {
       base.Init(context, treeNode);
-      foreach (var child in treeNode.MappedChildNodes) {
+      var nodes = treeNode.GetMappedChildNodes();
+      foreach (var child in nodes) {
         //don't add if it is null; it can happen that "statement" is a comment line and statement's node is null.
         // So to make life easier for language creator, we just skip if it is null
         if (child.AstNode != null) 

Irony.Interpreter/InterpretedLanguageGrammar.cs

@@ -25,8 +25,6 @@ public abstract class InterpretedLanguageGrammar : Grammar, ICanRunSample {
        // making the class abstract so it won't load into Grammar Explorer
     public InterpretedLanguageGrammar(bool caseSensitive)
       : base(caseSensitive) {
-      this.DefaultLiteralNodeType = typeof(LiteralValueNode);  //default node type for literals
-      this.DefaultIdentifierNodeType = typeof(IdentifierNode); 
       this.LanguageFlags = LanguageFlags.CreateAst;
     }
 

Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs

@@ -51,12 +51,12 @@ public ExpressionEvaluatorGrammar() : base(caseSensitive: false) {
       //String literal with embedded expressions  ------------------------------------------------------------------
       var stringLit = new StringLiteral("string", "\"", StringOptions.AllowsAllEscapes | StringOptions.IsTemplate);
       stringLit.AddStartEnd("'", StringOptions.AllowsAllEscapes | StringOptions.IsTemplate); 
-      stringLit.AstNodeType = typeof(StringTemplateNode);
+      stringLit.AstConfig.NodeType = typeof(StringTemplateNode);
       var Expr = new NonTerminal("Expr"); //declare it here to use in template definition 
       var templateSettings = new StringTemplateSettings(); //by default set to Ruby-style settings 
       templateSettings.ExpressionRoot = Expr; //this defines how to evaluate expressions inside template
       this.SnippetRoots.Add(Expr);
-      stringLit.AstData = templateSettings;
+      stringLit.AstConfig.Data = templateSettings;
       //--------------------------------------------------------------------------------------------------------
 
       // 2. Non-terminals

Irony.Tests/040.Irony.Tests.VsTest.2010.csproj

@@ -71,6 +71,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="CommentTerminalTests.cs" />
+    <Compile Include="ErrorRecoveryTests.cs" />
     <Compile Include="OperatorTests.cs" />
     <Compile Include="TestHelper.cs" />
     <Compile Include="TokenPreviewResolution\ConflictResolutionTests.cs" />

Irony.Tests/ErrorRecoveryTests.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Text.RegularExpressions;
+using Irony.Parsing;
+
+namespace Irony.Tests {
+#if USE_NUNIT
+    using NUnit.Framework;
+    using TestClass = NUnit.Framework.TestFixtureAttribute;
+    using TestMethod = NUnit.Framework.TestAttribute;
+    using TestInitialize = NUnit.Framework.SetUpAttribute;
+#else
+  using Microsoft.VisualStudio.TestTools.UnitTesting;
+#endif
+
+  [TestClass]
+  public class ErrorRecoveryTests {
+
+    #region Grammars
+    //A simple grammar for language consisting of simple assignment statements: x=y + z; z= t + m;
+    public class ErrorRecoveryGrammar : Grammar {
+      public ErrorRecoveryGrammar() {
+        var id = new IdentifierTerminal("id");
+        var expr = new NonTerminal("expr");
+        var stmt = new NonTerminal("stmt");
+        var stmtList = new NonTerminal("stmt");
+
+        base.Root = stmtList;
+        stmtList.Rule = MakeStarRule(stmtList, stmt);
+        stmt.Rule = id + "=" + expr + ";";
+        stmt.ErrorRule = SyntaxError + ";";
+        expr.Rule = id | id + "+" + id; 
+      }
+    }// class
+
+    #endregion
+
+    [TestMethod]
+    public void TestErrorRecovery() {
+
+      var grammar = new ErrorRecoveryGrammar();
+      var parser = new Parser(grammar);
+      TestHelper.CheckGrammarErrors(parser);
+
+      //correct sample
+      var parseTree = parser.Parse("x = y; y = z + m; m = n;");
+      Assert.IsFalse(parseTree.HasErrors(), "Unexpected parse errors in correct source sample.");
+
+      parseTree = parser.Parse("x = y; m = = d ; y = z + m; x = z z; m = n;");
+      Assert.AreEqual(2, parseTree.ParserMessages.Count, "Invalid # of errors.");
+
+    }
+
+
+  }//class
+}//namespace

Irony.Tests/TokenPreviewResolution/ConflictResolutionTests.cs

@@ -60,8 +60,8 @@ public void TestConflictGrammarWithHintsOnRules() {
       Assert.AreEqual("definition", term.Name);
 
       Assert.AreEqual(1, tree.Root.ChildNodes.Count);
-      var nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray();
-      Assert.AreEqual("fieldModifier", nodes[0].Term.Name);
+      var modNode = tree.Root.ChildNodes[0].ChildNodes[0];
+      Assert.AreEqual("fieldModifier", modNode.Term.Name);
 
       //Property 
       sample = PropertySample;
@@ -75,8 +75,8 @@ public void TestConflictGrammarWithHintsOnRules() {
       Assert.AreEqual("definition", term.Name);
 
       Assert.AreEqual(1, tree.Root.ChildNodes.Count);
-      nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray();
-      Assert.AreEqual("propModifier", nodes[0].Term.Name);
+      modNode = tree.Root.ChildNodes[0].ChildNodes[0];
+      Assert.AreEqual("propModifier", modNode.Term.Name);
     }
 
     //Hints on terms ---------------------------------------------------------------------
@@ -98,7 +98,7 @@ public void TestConflictGrammar_HintsOnTerms() {
       Assert.AreEqual("StatementList", term.Name);
 
       Assert.AreEqual(2, tree.Root.ChildNodes.Count);
-      var nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray();
+      var nodes = tree.Root.ChildNodes.Select(t => t.ChildNodes[0]).ToArray();
       Assert.AreEqual("fieldDef", nodes[0].Term.Name);
       Assert.AreEqual("fieldDef", nodes[1].Term.Name);
 
@@ -114,7 +114,7 @@ public void TestConflictGrammar_HintsOnTerms() {
       Assert.AreEqual("StatementList", term.Name);
 
       Assert.AreEqual(3, tree.Root.ChildNodes.Count);
-      nodes = tree.Root.ChildNodes.Select(t => t.FirstChild).ToArray();
+      nodes = tree.Root.ChildNodes.Select(t => t.ChildNodes[0]).ToArray();
       Assert.AreEqual("propDef", nodes[0].Term.Name);
       Assert.AreEqual("fieldDef", nodes[1].Term.Name);
       Assert.AreEqual("methodDef", nodes[2].Term.Name);

Irony/010.Irony.2010.csproj

@@ -76,7 +76,8 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Ast\AstBuilder.cs" />
-    <None Include="Ast\AstNodeConfig.cs" />
+    <Compile Include="Ast\AstExtensions.cs" />
+    <Compile Include="Ast\AstNodeConfig.cs" />
     <Compile Include="Ast\AstContext.cs" />
     <Compile Include="Ast\AstInterfaces.cs" />
     <Compile Include="Parsing\ConflictResolution\ImpliedPrecedenceHint.cs" />
@@ -125,7 +126,6 @@
     <Compile Include="Parsing\Data\ParserData.cs" />
     <Compile Include="Parsing\Parser\ParserStack.cs" />
     <Compile Include="Parsing\Parser\ParseTree.cs" />
-    <Compile Include="Parsing\Parser\CoreParser.cs" />
     <Compile Include="Parsing\Scanner\SourceLocation.cs" />
     <Compile Include="Parsing\Parser\ParserDataPrinter.cs" />
     <Compile Include="Parsing\Scanner\TokenEditorInfo.cs" />

Irony/101.IronySilverlight.2010.csproj

@@ -58,6 +58,7 @@
   <ItemGroup>
     <Compile Include="Ast\AstBuilder.cs" />
     <Compile Include="Ast\AstContext.cs" />
+    <Compile Include="Ast\AstExtensions.cs" />
     <Compile Include="Ast\AstInterfaces.cs" />
     <Compile Include="Ast\AstNodeConfig.cs" />
     <Compile Include="Parsing\ConflictResolution\ConditionalParserAction.cs" />
@@ -86,7 +87,6 @@
     <Compile Include="Parsing\Grammar\LanguageAttribute.cs" />
     <Compile Include="Parsing\Grammar\NonTerminal.cs" />
     <Compile Include="Parsing\Grammar\TermReportGroups.cs" />
-    <Compile Include="Parsing\Parser\CoreParser.cs" />
     <Compile Include="Parsing\Parser\Parser.cs" />
     <Compile Include="Parsing\Parser\ParserActions\AcceptParserAction.cs" />
     <Compile Include="Parsing\Parser\ParserActions\ErrorRecoveryParserAction.cs" />
@@ -144,7 +144,6 @@
     <EmbeddedResource Include="Resources.resx" />
   </ItemGroup>
   <ItemGroup>
-    <Content Include="Ast\_about.txt" />
     <Content Include="Parsing\Data\Construction\_about_parser_construction.txt" />
   </ItemGroup>
   <ItemGroup>