diff --git a/Irony.Interpreter/Ast/Base/AstNode.cs b/Irony.Interpreter/Ast/Base/AstNode.cs index da2134e..33ceac9 100644 --- a/Irony.Interpreter/Ast/Base/AstNode.cs +++ b/Irony.Interpreter/Ast/Base/AstNode.cs @@ -51,7 +51,7 @@ public partial class AstNode : IAstNodeInit, IBrowsableAstNode, IVisitableNode { // Node's parent can set it to "property name" or role of the child node in parent's node currentFrame.Context. public string Role; // Default AstNode.ToString() returns 'Role: AsString', which is used for showing node in AST tree. - public string AsString { get; protected set; } + public virtual string AsString { get; protected set; } public readonly AstNodeList ChildNodes = new AstNodeList(); //List of child nodes //Reference to Evaluate method implementation. Initially set to DoEvaluate virtual method. diff --git a/Irony.Interpreter/Ast/Statements/SpecialFormNode.cs b/Irony.Interpreter/Ast/Statements/SpecialFormNode.cs deleted file mode 100644 index e5527d9..0000000 --- a/Irony.Interpreter/Ast/Statements/SpecialFormNode.cs +++ /dev/null @@ -1,134 +0,0 @@ -#region License -/* ********************************************************************************** - * Copyright (c) Roman Ivantsov - * This source code is subject to terms and conditions of the MIT License - * for Irony. A copy of the license can be found in the License.txt file - * at the root of this distribution. - * By using this source code in any fashion, you are agreeing to be bound by the terms of the - * MIT License. - * You must not remove this notice from this software. - * **********************************************************************************/ -#endregion - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -using Irony.Ast; -using Irony.Parsing; - -namespace Irony.Interpreter.Ast { - - //A node representing special form - a statement handled by a SpecialForm method provided by Runtime. - public class SpecialFormNode : AstNode { - string _formName; - SpecialForm _specialForm; - AstNode[] _specialFormArgs; - - public override void Init(AstContext context, ParseTreeNode treeNode) { - base.Init(context, treeNode); - var nodes = treeNode.GetMappedChildNodes(); - TargetRef = AddChild("Target", nodes[0]); - TargetRef.UseType = NodeUseType.CallTarget; - _targetName = nodes[0].FindTokenAndGetText(); - Arguments = AddChild("Args", nodes[1]); - AsString = "Call " + _targetName; - context.Language. - } - - protected override object DoEvaluate(ScriptThread thread) { - thread.CurrentNode = this; //standard prolog - SetupEvaluateMethod(thread); - var result = Evaluate(thread); - thread.CurrentNode = Parent; //standard epilog - return result; - } - - private void SetupEvaluateMethod(ScriptThread thread) { - var languageTailRecursive = thread.Runtime.Language.Grammar.LanguageFlags.IsSet(LanguageFlags.TailRecursive); - lock (this.LockObject) { - _specialForm = null; - var bnd = thread.Bind(_formName, BindingRequestFlags.Invoke); - if (bnd != null) - _specialForm = bnd.GetValueRef(thread) as SpecialForm; - IBindingSource src; - var target = thread.Runtime.BuiltIns.TryGetValue(_formName, out src); - if (target is SpecialForm) { - _specialForm = target as SpecialForm; - _specialFormArgs = Arguments.ChildNodes.ToArray(); - this.Evaluate = EvaluateSpecialForm; - } else { - if (languageTailRecursive) { - var isTail = Flags.IsSet(AstNodeFlags.IsTail); - if (isTail) - this.Evaluate = EvaluateTail; - else - this.Evaluate = EvaluateWithTailCheck; - } else - this.Evaluate = EvaluateNoTail; - } - }//lock - } - - // Evaluation for special forms - private object EvaluateSpecialForm(ScriptThread thread) { - thread.CurrentNode = this; //standard prolog - var result = _specialForm(thread, _specialFormArgs); - thread.CurrentNode = Parent; //standard epilog - return result; - } - - - // Evaluation for non-tail languages - private object EvaluateNoTail(ScriptThread thread) { - thread.CurrentNode = this; //standard prolog - var target = TargetRef.Evaluate(thread); - var iCall = target as ICallTarget; - if (iCall == null) - thread.ThrowScriptError(Resources.ErrVarIsNotCallable, _targetName); - var args = (object[])Arguments.Evaluate(thread); - object result = iCall.Call(thread, args); - thread.CurrentNode = Parent; //standard epilog - return result; - } - - //Evaluation for tailed languages - private object EvaluateTail(ScriptThread thread) { - thread.CurrentNode = this; //standard prolog - var target = TargetRef.Evaluate(thread); - var iCall = target as ICallTarget; - if (iCall == null) - thread.ThrowScriptError(Resources.ErrVarIsNotCallable, _targetName); - var args = (object[])Arguments.Evaluate(thread); - thread.Tail = iCall; - thread.TailArgs = args; - thread.CurrentNode = Parent; //standard epilog - return null; - } - - private object EvaluateWithTailCheck(ScriptThread thread) { - thread.CurrentNode = this; //standard prolog - var target = TargetRef.Evaluate(thread); - var iCall = target as ICallTarget; - if (iCall == null) - thread.ThrowScriptError(Resources.ErrVarIsNotCallable, _targetName); - var args = (object[])Arguments.Evaluate(thread); - object result = null; - result = iCall.Call(thread, args); - //Note that after invoking tail we can get another tail. - // So we need to keep calling tails while they are there. - while (thread.Tail != null) { - var tail = thread.Tail; - var tailArgs = thread.TailArgs; - thread.Tail = null; - thread.TailArgs = null; - result = tail.Call(thread, tailArgs); - } - thread.CurrentNode = Parent; //standard epilog - return result; - } - - }//class - -}//namespace diff --git a/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs b/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs index 1b0eac0..7e2b26f 100644 --- a/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs +++ b/Irony.Interpreter/_Evaluator/ExpressionEvaluatorGrammar.cs @@ -85,7 +85,7 @@ public ExpressionEvaluatorGrammar() : base(caseSensitive: false) { Term.Rule = number | ParExpr | stringLit | FunctionCall | identifier | MemberAccess | IndexedAccess; ParExpr.Rule = "(" + Expr + ")"; UnExpr.Rule = UnOp + Term + ReduceHere(); - UnOp.Rule = ToTerm("+") | "-"; + UnOp.Rule = ToTerm("+") | "-" | "!"; BinExpr.Rule = Expr + BinOp + Expr; BinOp.Rule = ToTerm("+") | "-" | "*" | "/" | "**" | "==" | "<" | "<=" | ">" | ">=" | "!=" | "&&" | "||" | "&" | "|"; PrefixIncDec.Rule = IncDecOp + identifier; @@ -113,6 +113,7 @@ public ExpressionEvaluatorGrammar() : base(caseSensitive: false) { RegisterOperators(30, "+", "-"); RegisterOperators(40, "*", "/"); RegisterOperators(50, Associativity.Right, "**"); + RegisterOperators(60, "!"); // For precedence to work, we need to take care of one more thing: BinOp. //For BinOp which is or-combination of binary operators, we need to either // 1) mark it transient or 2) set flag TermFlags.InheritPrecedence diff --git a/Irony.Samples/GWBasic/GwBasicGrammar.cs b/Irony.Samples/GWBasic/GwBasicGrammar.cs index 032ac6e..ec95e25 100644 --- a/Irony.Samples/GWBasic/GwBasicGrammar.cs +++ b/Irony.Samples/GWBasic/GwBasicGrammar.cs @@ -217,13 +217,12 @@ and then checks the terminals agains the ExpectedTerms set in the parser state. this.LanguageFlags = LanguageFlags.NewLineBeforeEOF; - lineNumber.ValidateToken += identifier_ValidateToken; - + lineNumber.ValidateToken += lineNumber_ValidateToken; } - void identifier_ValidateToken(object sender, ParsingEventArgs e) { + void lineNumber_ValidateToken(object sender, ParsingEventArgs e) { if (e.Context.CurrentToken.ValueString.Length > 4) - e.Context.CurrentToken = e.Context.CreateErrorToken("Identifier cannot be longer than 4 characters"); + e.Context.CurrentToken = e.Context.CreateErrorToken("Line number cannot be longer than 4 characters"); }//constructor }//class diff --git a/Irony.Samples/MiniPython/MiniPython.cs b/Irony.Samples/MiniPython/MiniPython.cs index 215fa33..4f470e0 100644 --- a/Irony.Samples/MiniPython/MiniPython.cs +++ b/Irony.Samples/MiniPython/MiniPython.cs @@ -70,7 +70,7 @@ public MiniPythonGrammar() : base(caseSensitive: true) { BinExpr.Rule = Expr + BinOp + Expr; BinOp.Rule = ToTerm("+") | "-" | "*" | "/" | "**"; AssignmentStmt.Rule = identifier + "=" + Expr; - Stmt.Rule = AssignmentStmt | Expr | ReturnStmt; + Stmt.Rule = AssignmentStmt | Expr | ReturnStmt | Empty; ReturnStmt.Rule = "return" + Expr; //Not supported for execution! - we associate NotSupportedNode with ReturnStmt //Eos is End-Of-Statement token produced by CodeOutlineFilter ExtStmt.Rule = Stmt + Eos | FunctionDef; diff --git a/Irony.Samples/SourceSamples/SampleExpressionEvaluator/TestIif.txt b/Irony.Samples/SourceSamples/SampleExpressionEvaluator/TestIif.txt index b2bf6aa..e8230ef 100644 --- a/Irony.Samples/SourceSamples/SampleExpressionEvaluator/TestIif.txt +++ b/Irony.Samples/SourceSamples/SampleExpressionEvaluator/TestIif.txt @@ -1,3 +1,4 @@ # Testing special form "IIF" -# You can also use equivalent "?" operator: '1 > 0 ? "true" : 1/0' -iif(1 > 0, "true", 1/0) # notice that if-false argument is not evaluated +# and equivalent "?" operator: '1 > 0 ? "true" : 1/0' +# notice that if-false argument is not evaluated in both cases +iif(1 > 0, "true", 1/0) + ", " + (1 > 0 ? "true" : 1/0) # expected: true, true \ No newline at end of file diff --git a/Irony.Samples/SourceSamples/c#/escapes_in_IDs.cs b/Irony.Samples/SourceSamples/c#/escapes_in_IDs.cs index 3979ee9..e3c2175 100644 --- a/Irony.Samples/SourceSamples/c#/escapes_in_IDs.cs +++ b/Irony.Samples/SourceSamples/c#/escapes_in_IDs.cs @@ -1,25 +1,23 @@ //example from c# spec, page 93 namespace Sample { - class @class - { - public static void @static(bool @bool) { - if (@bool) - System.Console.WriteLine("true"); - else - System.Console.WriteLine("false"); - } - } - class Class1 - { - static void M() { - cl\u0061ss.st\u0061tic(true); - } - } + class @class { + public static void @static(bool @bool) { + if (@bool) + System.Console.WriteLine("true"); + else + System.Console.WriteLine("false"); + } + } + class Class1 { + static void M() { + cl\u0061ss.st\u0061tic(true); + } + } } /* code comment from spec: -[the code] defines a class named “class” with a static method named “static” that takes a parameter named -“bool”. Note that since Unicode escapes are not permitted in keywords, the token “cl\u0061ss” is an -identifier, and is the same identifier as “@class”. end example] +[the code] defines a class named 'class' with a static method named 'static' that takes a parameter named +'bool'. Note that since Unicode escapes are not permitted in keywords, the token 'cl\u0061ss' is an +identifier, and is the same identifier as '@class'. end example] */ \ No newline at end of file diff --git a/Irony/Parsing/Data/Construction/_about_parser_construction.txt b/Irony/Parsing/Data/Construction/_about_parser_construction.txt index f6eed86..0004b91 100644 --- a/Irony/Parsing/Data/Construction/_about_parser_construction.txt +++ b/Irony/Parsing/Data/Construction/_about_parser_construction.txt @@ -19,7 +19,7 @@ through Srongly-Connected Components (SCC) algorithm would not be much faster. With immediate propagation we attempt to add a transition to Includes set of another transition only once and stop propagation of the transition further down the chain if it is already there. Essentially, we don't waste time propagating sets of transitions through chains of Includes if the transitions are - alredy there, propagated through different route. This is what SCC method is trying to mitigate - repeated propagation of transitions - + already there, propagated through different route. This is what SCC method is trying to mitigate - repeated propagation of transitions - but this is not happening in our implementation. Maybe I'm mistaken, this is a guess, not a formal proof - let me know if you see any flaws in my reasoning. @@ -41,5 +41,5 @@ current state has a single reduce action (DefaultReduceAction property is not null). We do not read next token because it is not needed for finding an action - there is one single possible action anyway. As a result the Scanner would never start scanning a new token when parser in this single-reduce state - and therefore scanner would not invoke the parser-scanner link. - See CoreParser.ExecuteAction method for details. + See Parser.ExecuteAction method for details. diff --git a/Irony/Parsing/Grammar/Grammar.cs b/Irony/Parsing/Grammar/Grammar.cs index 264607b..0e94613 100644 --- a/Irony/Parsing/Grammar/Grammar.cs +++ b/Irony/Parsing/Grammar/Grammar.cs @@ -222,6 +222,7 @@ public virtual bool IsWhitespaceOrDelimiter(char ch) { switch (ch) { case ' ': case '\t': case '\r': case '\n': case '\v': //whitespaces case '(': case ')': case ',': case ';': case '[': case ']': case '{': case '}': + case (char)0: //EOF return true; default: return false; @@ -282,27 +283,14 @@ public BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter return MakeListRule(listNonTerminal, delimiter, listMember, TermListOptions.StarList); } - //Note: Here and in other make-list methods with delimiter. More logical would be the parameters order (list, listMember, delimiter=null). - // But for historical reasons it's the way it is, and I think it's too late to change and to reverse the order of delimiter and listMember. - // Too many existing grammars would be broken. The big trouble is that these two parameters are of the same type, so compiler would not - // detect that order had changed (if we change it) for existing grammars. The grammar would stop working at runtime, and it would - // require some effort to debug and find the cause of the problem. For these reasons, we leave it as is. - [Obsolete("Method overload is obsolete - use MakeListRule instead")] - public BnfExpression MakePlusRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember, TermListOptions options) { - return MakeListRule(listNonTerminal, delimiter, listMember, options); - } - - [Obsolete("Method overload is obsolete - use MakeListRule instead")] - public BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember, TermListOptions options) { - return MakeListRule(listNonTerminal, delimiter, listMember, options | TermListOptions.StarList); - } - protected BnfExpression MakeListRule(NonTerminal list, BnfTerm delimiter, BnfTerm listMember, TermListOptions options = TermListOptions.PlusList) { //If it is a star-list (allows empty), then we first build plus-list - var isStarList = options.IsSet(TermListOptions.AllowEmpty); - NonTerminal plusList = isStarList ? new NonTerminal(listMember.Name + "+") : list; - //"list" is the real list for which we will construct expression - it is either extra plus-list or original listNonTerminal. - // In the latter case we will use it later to construct expression for listNonTerminal + var isPlusList = !options.IsSet(TermListOptions.AllowEmpty); + var allowTrailingDelim = options.IsSet(TermListOptions.AllowTrailingDelimiter) && delimiter != null; + //"plusList" is the list for which we will construct expression - it is either extra plus-list or original list. + // In the former case (extra plus-list) we will use it later to construct expression for list + NonTerminal plusList = isPlusList ? list : new NonTerminal(listMember.Name + "+"); + plusList.SetFlag(TermFlags.IsList); plusList.Rule = plusList; // rule => list if (delimiter != null) plusList.Rule += delimiter; // rule => list + delim @@ -310,19 +298,19 @@ protected BnfExpression MakeListRule(NonTerminal list, BnfTerm delimiter, BnfTer plusList.Rule += PreferShiftHere(); // rule => list + delim + PreferShiftHere() plusList.Rule += listMember; // rule => list + delim + PreferShiftHere() + elem plusList.Rule |= listMember; // rule => list + delim + PreferShiftHere() + elem | elem - //trailing delimiter - if (options.IsSet(TermListOptions.AllowTrailingDelimiter) & delimiter != null) - plusList.Rule |= list + delimiter; // => list + delim + PreferShiftHere() + elem | elem | list + delim - // set Rule value - plusList.SetFlag(TermFlags.IsList); - //If we do not use exra list - we're done, return list.Rule - if (plusList == list) - return list.Rule; - // Let's setup listNonTerminal.Rule using plus-list we just created - //If we are here, TermListOptions.AllowEmpty is set, so we have star-list - list.Rule = Empty | plusList; - plusList.SetFlag(TermFlags.NoAstNode); - list.SetFlag(TermFlags.IsListContainer); //indicates that real list is one level lower + if (isPlusList) { + // if we build plus list - we're almost done; plusList == list + // add trailing delimiter if necessary; for star list we'll add it to final expression + if (allowTrailingDelim) + plusList.Rule |= list + delimiter; // rule => list + delim + PreferShiftHere() + elem | elem | list + delim + } else { + // Setup list.Rule using plus-list we just created + list.Rule = Empty | plusList; + if (allowTrailingDelim) + list.Rule |= plusList + delimiter | delimiter; + plusList.SetFlag(TermFlags.NoAstNode); + list.SetFlag(TermFlags.IsListContainer); //indicates that real list is one level lower + } return list.Rule; }//method #endregion @@ -427,6 +415,9 @@ private IEnumerable SymbolsToTerms(IEnumerable symbols) { // as a lookahead to Root non-terminal public readonly Terminal Eof = new Terminal("EOF", TokenCategory.Outline); + //Artificial terminal to use for injected/replaced tokens that must be ignored by parser. + public readonly Terminal Skip = new Terminal("(SKIP)", TokenCategory.Outline, TermFlags.IsNonGrammar); + //Used as a "line-start" indicator public readonly Terminal LineStartTerminal = new Terminal("LINE_START", TokenCategory.Outline); diff --git a/Irony/Parsing/Scanner/Token.cs b/Irony/Parsing/Scanner/Token.cs index 1fdd281..200f3b8 100644 --- a/Irony/Parsing/Scanner/Token.cs +++ b/Irony/Parsing/Scanner/Token.cs @@ -91,6 +91,10 @@ public override string ToString() { //Some terminals may need to return a bunch of tokens in one call to TryMatch; MultiToken is a container for these tokens public class MultiToken : Token { public TokenList ChildTokens; + + public MultiToken(params Token[] tokens) : this(tokens[0].Terminal, tokens[0].Location, new TokenList()) { + ChildTokens.AddRange(tokens); + } public MultiToken(Terminal term, SourceLocation location, TokenList childTokens) : base(term, location, string.Empty, null) { ChildTokens = childTokens; } diff --git a/Irony/Parsing/Terminals/ConstantTerminal.cs b/Irony/Parsing/Terminals/ConstantTerminal.cs index f07f8d9..f6bc7b9 100644 --- a/Irony/Parsing/Terminals/ConstantTerminal.cs +++ b/Irony/Parsing/Terminals/ConstantTerminal.cs @@ -21,9 +21,11 @@ namespace Irony.Parsing { public class ConstantsTable : Dictionary { } public class ConstantTerminal : Terminal { public readonly ConstantsTable Constants = new ConstantsTable(); - public ConstantTerminal(string name, Type nodeType) : base(name) { + public ConstantTerminal(string name, Type nodeType = null) : base(name) { base.SetFlag(TermFlags.IsConstant); - base.AstConfig.NodeType = nodeType; + if (nodeType != null) + base.AstConfig.NodeType = nodeType; + this.Priority = TerminalPriority.High; //constants have priority over normal identifiers } public void Add(string lexeme, object value) { @@ -39,15 +41,19 @@ public override void Init(GrammarData grammarData) { public override Token TryMatch(ParsingContext context, ISourceStream source) { string text = source.Text; foreach (var entry in Constants) { + source.PreviewPosition = source.Position; var constant = entry.Key; if (source.PreviewPosition + constant.Length > text.Length) continue; if (source.MatchSymbol(constant)) { source.PreviewPosition += constant.Length; + if (!this.Grammar.IsWhitespaceOrDelimiter(source.PreviewChar)) + continue; //make sure it is delimiter return source.CreateToken(this.OutputTerminal, entry.Value); } } return null; } + public override IList GetFirsts() { string[] array = new string[Constants.Count]; Constants.Keys.CopyTo(array, 0); diff --git a/Irony/Parsing/Terminals/IdentifierTerminal.cs b/Irony/Parsing/Terminals/IdentifierTerminal.cs index f0f6928..d41c3f3 100644 --- a/Irony/Parsing/Terminals/IdentifierTerminal.cs +++ b/Irony/Parsing/Terminals/IdentifierTerminal.cs @@ -27,7 +27,7 @@ namespace Irony.Parsing { public enum IdOptions : short { None = 0, AllowsEscapes = 0x01, - CanStartWithEscape = 0x03, //bit 2 with bit 1 together + CanStartWithEscape = 0x03, IsNotKeyword = 0x10, NameIncludesPrefix = 0x20,