minor fixes

Irony.Interpreter/Ast/Base/AstNode.cs

@@ -51,7 +51,7 @@ public partial class AstNode : IAstNodeInit, IBrowsableAstNode, IVisitableNode {
     // Node's parent can set it to "property name" or role of the child node in parent's node currentFrame.Context. 
     public string Role;
     // Default AstNode.ToString() returns 'Role: AsString', which is used for showing node in AST tree. 
-    public string AsString { get; protected set; }
+    public virtual string AsString { get; protected set; }
     public readonly AstNodeList ChildNodes = new AstNodeList();  //List of child nodes
 
     //Reference to Evaluate method implementation. Initially set to DoEvaluate virtual method. 

Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs

@@ -85,7 +85,7 @@ public ExpressionEvaluatorGrammar() : base(caseSensitive: false) {
       Term.Rule = number | ParExpr | stringLit | FunctionCall | identifier | MemberAccess | IndexedAccess;
       ParExpr.Rule = "(" + Expr + ")";
       UnExpr.Rule = UnOp + Term + ReduceHere();
-      UnOp.Rule = ToTerm("+") | "-"; 
+      UnOp.Rule = ToTerm("+") | "-" | "!"; 
       BinExpr.Rule = Expr + BinOp + Expr;
       BinOp.Rule = ToTerm("+") | "-" | "*" | "/" | "**" | "==" | "<" | "<=" | ">" | ">=" | "!=" | "&&" | "||" | "&" | "|";
       PrefixIncDec.Rule = IncDecOp + identifier;
@@ -113,6 +113,7 @@ public ExpressionEvaluatorGrammar() : base(caseSensitive: false) {
       RegisterOperators(30, "+", "-");
       RegisterOperators(40, "*", "/");
       RegisterOperators(50, Associativity.Right, "**");
+      RegisterOperators(60, "!");
       // For precedence to work, we need to take care of one more thing: BinOp. 
       //For BinOp which is or-combination of binary operators, we need to either 
       // 1) mark it transient or 2) set flag TermFlags.InheritPrecedence

Irony.Samples/GWBasic/GwBasicGrammar.cs

@@ -217,13 +217,12 @@ and then checks the terminals agains the ExpectedTerms set in the parser state.
 
             this.LanguageFlags = LanguageFlags.NewLineBeforeEOF;
 
-            lineNumber.ValidateToken += identifier_ValidateToken;
-
+            lineNumber.ValidateToken += lineNumber_ValidateToken;
       }
 
-        void identifier_ValidateToken(object sender, ParsingEventArgs e) {
+        void lineNumber_ValidateToken(object sender, ParsingEventArgs e) {
           if (e.Context.CurrentToken.ValueString.Length > 4)
-            e.Context.CurrentToken = e.Context.CreateErrorToken("Identifier cannot be longer than 4 characters"); 
+            e.Context.CurrentToken = e.Context.CreateErrorToken("Line number cannot be longer than 4 characters"); 
         }//constructor
 
     }//class

Irony.Samples/MiniPython/MiniPython.cs

@@ -70,7 +70,7 @@ public MiniPythonGrammar() : base(caseSensitive: true) {
       BinExpr.Rule = Expr + BinOp + Expr;
       BinOp.Rule = ToTerm("+") | "-" | "*" | "/" | "**";
       AssignmentStmt.Rule = identifier + "=" + Expr;
-      Stmt.Rule = AssignmentStmt | Expr | ReturnStmt;
+      Stmt.Rule = AssignmentStmt | Expr | ReturnStmt | Empty;
       ReturnStmt.Rule = "return" + Expr; //Not supported for execution! - we associate NotSupportedNode with ReturnStmt
       //Eos is End-Of-Statement token produced by CodeOutlineFilter
       ExtStmt.Rule = Stmt + Eos | FunctionDef;

Irony.Samples/SourceSamples/SampleExpressionEvaluator/TestIif.txt

@@ -1,3 +1,4 @@
 # Testing special form "IIF"
-# You can also use equivalent "?" operator: '1 > 0 ? "true" : 1/0'
-iif(1 > 0, "true", 1/0)  # notice that if-false argument is not evaluated
+# and equivalent "?" operator: '1 > 0 ? "true" : 1/0'
+# notice that if-false argument is not evaluated in both cases
+iif(1 > 0, "true", 1/0)  + ", " + (1 > 0 ? "true" : 1/0)   # expected: true, true

Irony.Samples/SourceSamples/c#/escapes_in_IDs.cs

@@ -1,25 +1,23 @@
 //example from c# spec, page 93
 namespace Sample {
-	class @class 
-	{ 
-	  public static void @static(bool @bool) { 
-	  if (@bool) 
-	   System.Console.WriteLine("true"); 
-	  else 
-	   System.Console.WriteLine("false"); 
-	 }  
-	} 
-	class Class1 
-	{ 
-	  static void M() { 
-	  cl\u0061ss.st\u0061tic(true); 
-	 } 
-	}
+  class @class {
+    public static void @static(bool @bool) {
+      if (@bool)
+        System.Console.WriteLine("true");
+      else
+        System.Console.WriteLine("false");
+    }
+  }
+  class Class1 {
+    static void M() {
+      cl\u0061ss.st\u0061tic(true);
+    }
+  }
 
 }
 
 /*  code comment from spec:
-[the code]  defines a class named “class” with a static method named “static” that takes a parameter named 
-“bool”. Note that since Unicode escapes are not permitted in keywords, the token “cl\u0061ss” is an 
-identifier, and is the same identifier as “@class”. end example]
+[the code]  defines a class named 'class' with a static method named 'static' that takes a parameter named 
+'bool'. Note that since Unicode escapes are not permitted in keywords, the token 'cl\u0061ss' is an 
+identifier, and is the same identifier as '@class'. end example]
 */

Irony/Parsing/Data/Construction/_about_parser_construction.txt

@@ -19,7 +19,7 @@
   through Srongly-Connected Components (SCC) algorithm would not be much faster. With immediate propagation we attempt to add 
   a transition to Includes set of another transition only once and stop propagation of the transition further down the chain if it is
   already there. Essentially, we don't waste time propagating sets of transitions through chains of Includes if the transitions are 
-  alredy there, propagated through different route. This is what SCC method is trying to mitigate - repeated propagation of transitions - 
+  already there, propagated through different route. This is what SCC method is trying to mitigate - repeated propagation of transitions - 
   but this is not happening in our implementation. Maybe I'm mistaken, this is a guess, not a formal proof - let me know if you see 
   any flaws in my reasoning.
 
@@ -41,5 +41,5 @@
   current state has a single reduce action (DefaultReduceAction property is not null). We do not read next token because it is not needed
   for finding an action - there is one single possible action anyway. As a result the Scanner would never start scanning a new token 
   when parser in this single-reduce state - and therefore scanner would not invoke the parser-scanner link. 
-  See CoreParser.ExecuteAction method for details.
+  See Parser.ExecuteAction method for details.
 

Irony/Parsing/Grammar/Grammar.cs

@@ -222,6 +222,7 @@ public virtual bool IsWhitespaceOrDelimiter(char ch) {
       switch (ch) {
         case ' ':   case '\t': case '\r': case '\n': case '\v': //whitespaces
         case '(': case ')': case ',': case ';': case '[': case ']': case '{': case '}':
+        case (char)0: //EOF
           return true;
         default:
           return false;
@@ -282,47 +283,34 @@ public BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter
       return MakeListRule(listNonTerminal, delimiter, listMember, TermListOptions.StarList);
     }
 
-    //Note: Here and in other make-list methods with delimiter. More logical would be the parameters order (list, listMember, delimiter=null).
-    // But for historical reasons it's the way it is, and I think it's too late to change and to reverse the order of delimiter and listMember.
-    // Too many existing grammars would be broken. The big trouble is that these two parameters are of the same type, so compiler would not 
-    // detect that order had changed (if we change it) for existing grammars. The grammar would stop working at runtime, and it would 
-    // require some effort to debug and find the cause of the problem. For these reasons, we leave it as is. 
-    [Obsolete("Method overload is obsolete - use MakeListRule instead")]
-    public BnfExpression MakePlusRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember, TermListOptions options) {
-      return MakeListRule(listNonTerminal, delimiter, listMember, options);
-   }
-
-    [Obsolete("Method overload is obsolete - use MakeListRule instead")]
-    public BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember, TermListOptions options) {
-      return MakeListRule(listNonTerminal, delimiter, listMember, options | TermListOptions.StarList);
-    }
-
     protected BnfExpression MakeListRule(NonTerminal list, BnfTerm delimiter, BnfTerm listMember, TermListOptions options = TermListOptions.PlusList) {
       //If it is a star-list (allows empty), then we first build plus-list
-      var isStarList = options.IsSet(TermListOptions.AllowEmpty);
-      NonTerminal plusList = isStarList ? new NonTerminal(listMember.Name + "+") : list;
-      //"list" is the real list for which we will construct expression - it is either extra plus-list or original listNonTerminal. 
-      // In the latter case we will use it later to construct expression for listNonTerminal
+      var isPlusList = !options.IsSet(TermListOptions.AllowEmpty);
+      var allowTrailingDelim = options.IsSet(TermListOptions.AllowTrailingDelimiter) && delimiter != null;
+      //"plusList" is the list for which we will construct expression - it is either extra plus-list or original list. 
+      // In the former case (extra plus-list) we will use it later to construct expression for list
+      NonTerminal plusList = isPlusList ? list : new NonTerminal(listMember.Name + "+");
+      plusList.SetFlag(TermFlags.IsList);
       plusList.Rule = plusList;  // rule => list
       if (delimiter != null)
         plusList.Rule += delimiter;  // rule => list + delim
       if (options.IsSet(TermListOptions.AddPreferShiftHint))
         plusList.Rule += PreferShiftHere(); // rule => list + delim + PreferShiftHere()
       plusList.Rule += listMember;          // rule => list + delim + PreferShiftHere() + elem
       plusList.Rule |= listMember;        // rule => list + delim + PreferShiftHere() + elem | elem
-      //trailing delimiter
-      if (options.IsSet(TermListOptions.AllowTrailingDelimiter) & delimiter != null)
-        plusList.Rule |= list + delimiter; // => list + delim + PreferShiftHere() + elem | elem | list + delim
-      // set Rule value
-      plusList.SetFlag(TermFlags.IsList);
-      //If we do not use exra list - we're done, return list.Rule
-      if (plusList == list) 
-        return list.Rule;
-      // Let's setup listNonTerminal.Rule using plus-list we just created
-      //If we are here, TermListOptions.AllowEmpty is set, so we have star-list
-      list.Rule = Empty | plusList;
-      plusList.SetFlag(TermFlags.NoAstNode); 
-      list.SetFlag(TermFlags.IsListContainer); //indicates that real list is one level lower
+      if (isPlusList) {
+        // if we build plus list - we're almost done; plusList == list
+        // add trailing delimiter if necessary; for star list we'll add it to final expression
+        if (allowTrailingDelim)
+          plusList.Rule |= list + delimiter; // rule => list + delim + PreferShiftHere() + elem | elem | list + delim
+      } else {
+        // Setup list.Rule using plus-list we just created
+        list.Rule = Empty | plusList;
+        if (allowTrailingDelim)
+          list.Rule |= plusList + delimiter | delimiter;
+        plusList.SetFlag(TermFlags.NoAstNode);
+        list.SetFlag(TermFlags.IsListContainer); //indicates that real list is one level lower
+      } 
       return list.Rule; 
     }//method
     #endregion
@@ -427,6 +415,9 @@ private IEnumerable<Terminal> SymbolsToTerms(IEnumerable<string> symbols) {
     // as a lookahead to Root non-terminal
     public readonly Terminal Eof = new Terminal("EOF", TokenCategory.Outline);
 
+    //Artificial terminal to use for injected/replaced tokens that must be ignored by parser. 
+    public readonly Terminal Skip = new Terminal("(SKIP)", TokenCategory.Outline, TermFlags.IsNonGrammar);
+
     //Used as a "line-start" indicator
     public readonly Terminal LineStartTerminal = new Terminal("LINE_START", TokenCategory.Outline);
 

Irony/Parsing/Scanner/Token.cs

@@ -91,6 +91,10 @@ public override string ToString() {
   //Some terminals may need to return a bunch of tokens in one call to TryMatch; MultiToken is a container for these tokens
   public class MultiToken : Token {
     public TokenList ChildTokens;
+
+    public MultiToken(params Token[] tokens) : this(tokens[0].Terminal, tokens[0].Location, new TokenList()) {
+        ChildTokens.AddRange(tokens);
+    }
     public MultiToken(Terminal term, SourceLocation location, TokenList childTokens) : base(term, location, string.Empty, null) {
       ChildTokens = childTokens;
     }

Irony/Parsing/Terminals/ConstantTerminal.cs

@@ -21,9 +21,11 @@ namespace Irony.Parsing {
   public class ConstantsTable : Dictionary<string, object> { }
   public class ConstantTerminal : Terminal {
     public readonly ConstantsTable Constants = new ConstantsTable();
-    public ConstantTerminal(string name, Type nodeType) : base(name) {
+    public ConstantTerminal(string name, Type nodeType = null) : base(name) {
       base.SetFlag(TermFlags.IsConstant);
-      base.AstConfig.NodeType = nodeType;
+      if (nodeType != null)
+        base.AstConfig.NodeType = nodeType;
+      this.Priority = TerminalPriority.High; //constants have priority over normal identifiers
     }
 
     public void Add(string lexeme, object value) {
@@ -39,15 +41,19 @@ public override void Init(GrammarData grammarData) {
     public override Token TryMatch(ParsingContext context, ISourceStream source) {
       string text = source.Text;
       foreach (var entry in Constants) {
+        source.PreviewPosition = source.Position;
         var constant = entry.Key;
         if (source.PreviewPosition + constant.Length > text.Length) continue;
         if (source.MatchSymbol(constant)) {
           source.PreviewPosition += constant.Length;
+          if (!this.Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
+            continue; //make sure it is delimiter
           return source.CreateToken(this.OutputTerminal, entry.Value);
         }
       }
       return null;
     }
+
     public override IList<string> GetFirsts() {
       string[] array = new string[Constants.Count];
       Constants.Keys.CopyTo(array, 0);

Irony/Parsing/Terminals/IdentifierTerminal.cs

@@ -27,7 +27,7 @@ namespace Irony.Parsing {
   public enum IdOptions : short {
     None = 0,
     AllowsEscapes = 0x01,
-    CanStartWithEscape = 0x03, //bit 2 with bit 1 together  
+    CanStartWithEscape = 0x03, 
 
     IsNotKeyword = 0x10,
     NameIncludesPrefix = 0x20,