Merge pull request #519 from b3b00/feature/subnodenames

sub node names attributes
b3b00 · Dec 18, 2024 · ee5352e · ee5352e
2 parents 3ec044a + 72ed3b9
commit ee5352e
Show file tree

Hide file tree

Showing 12 changed files with 154 additions and 26 deletions.
diff --git a/src/samples/XML/MinimalXmlParser.cs b/src/samples/XML/MinimalXmlParser.cs
@@ -30,16 +30,16 @@ public string Document(List<string> startingMiscs, string root, List<string> end
             return b.ToString();
         }
 
-        [Production("element : OPEN[d] ID attributes SLASH[d] CLOSE[d]")]
-        public string AutoElement(Token<MinimalXmlLexer> id, string attributes)
+        [Production("element : OPEN[d] ID attribute* SLASH[d] CLOSE[d]")]
+        public string AutoElement(Token<MinimalXmlLexer> id, List<string> attributes)
         {
-            return $"autoTag({id.Value}, {attributes})";
+            return $"autoTag({id.Value}, {string.Join(", ",attributes.Select(x => x.ToString()))})";
         }
 
-        [Production("opentag : OPEN[d] ID attributes CLOSE[d]")]
-        public string OpenTag(Token<MinimalXmlLexer> tagName, string attributes)
+        [Production("opentag : OPEN[d] ID attribute* CLOSE[d]")]
+        public string OpenTag(Token<MinimalXmlLexer> tagName, List<string> attributes)
         {
-            return $"open ({tagName.Value}, {attributes})";
+            return $"open ({tagName.Value}, {string.Join(", ",attributes.Select(x => x.ToString()))})";
         }
 
         [Production("closetag : OPEN[d] SLASH[d] ID CLOSE[d]")]
@@ -48,6 +48,7 @@ public string CloseTag(Token<MinimalXmlLexer> id)
             return $"close({id.Value})";
         }
 
+        [SubNodeNames(null, "elements",null)]
         [Production("element : opentag [element|pi|comment|content]* closetag")]
         public string CompoundElement(string open, List<string> subs, string close)
         {
@@ -67,18 +68,12 @@ public string Comment(Token<MinimalXmlLexer> comment)
             return $"comment({comment.Value})";
         }
 
-        [Production("pi : OPEN_PI[d] ID attributes CLOSE_PI[d]")]
-        public string Pi(Token<MinimalXmlLexer> id , string attributes)
+        [Production("pi : OPEN_PI[d] ID attribute* CLOSE_PI[d]")]
+        public string Pi(Token<MinimalXmlLexer> id , List<string> attributes)
         {
-            return $"pi({id.Value} :: {attributes.ToString()})";
+            return $"pi({id.Value} :: {string.Join(", ",attributes.Select(x => x.ToString()))})";
         }
 
-
-        [Production("attributes : attribute*")]
-        public string Attributes(List<string> attributes)
-        {
-            return string.Join(", ",attributes.Select(x => x.ToString()));
-        }
 
         [Production("attribute: ID EQUALS[d] VALUE")]
         public string Attribute(Token<MinimalXmlLexer> id, Token<MinimalXmlLexer> value)

diff --git a/src/samples/XML/xml.grammar b/src/samples/XML/xml.grammar
@@ -0,0 +1,62 @@
+
+genericLexer MinimalXmlLexer;
+
+[Mode]
+[Push("tag")]
+[Sugar] OPEN : "<";
+
+[UpTo] CONTENT : "<"; 
+
+[Mode]
+[Push("pi")]
+[Sugar] OPEN_PI : "<?";
+
+[Mode]
+[MultiLineComment] COMMENT : "<!--" "-->";
+
+[Mode("tag")]
+[Mode("pi")]
+[AlphaId] ID;
+
+[Mode("tag")]
+[Sugar] SLASH : "/";
+
+[Mode("tag", "pi")]
+[Sugar] EQUALS : "=";
+
+[Mode("tag", "pi")]
+[Mode("pi")]
+[String] VALUE;
+
+[Mode("pi")]
+[Pop]
+[Sugar] CLOSE_PI : "?>";
+
+[Mode("tag")]
+[Pop]
+[Sugar] CLOSE : ">";
+
+
+
+parser MinimalXmlParser;
+
+-> document : misc* element misc*;
+
+element : OPEN[d] ID attribute* SLASH[d] CLOSE[d];
+
+opentag : OPEN[d] ID attribute* CLOSE[d];
+
+closetag : OPEN[d] SLASH[d] ID CLOSE[d];
+
+element : opentag [element|pi|comment|content]* closetag;
+
+misc : [comment | pi | content];
+
+comment : COMMENT;
+
+pi : OPEN_PI[d] ID attribute* CLOSE_PI[d];
+
+attribute: ID EQUALS[d] VALUE;
+
+content : CONTENT;
+
diff --git a/src/sly/lexer/TokenChannel.cs b/src/sly/lexer/TokenChannel.cs
@@ -50,11 +50,9 @@ public Token<IN> this[int key]
             set => SetToken(key,value);
         }
 
-        public void Add(Token<IN> token)
+        public void Shift()
         {
-            Tokens.Add(token);
-            if (token != null) 
-                token.PositionInTokenFlow = Tokens.Count;
+            Tokens.Add(null);
         }
 
         [ExcludeFromCodeCoverage]

diff --git a/src/sly/lexer/TokenChannels.cs b/src/sly/lexer/TokenChannels.cs
@@ -82,7 +82,7 @@ public void Add(Token<IN> token)
                 }
                 for (int i = 0; i < shift; i++)
                 {
-                    channel.Add(null);
+                    channel.Shift();
                 }
                 _tokenChannels[token.Channel] = channel;
             }
@@ -92,7 +92,7 @@ public void Add(Token<IN> token)
             {
                 for (int i = channel.Count; i < index; i++)
                 {
-                    channel.Add(null);
+                    channel.Shift();
                 }
 
                 if (channel.ChannelId == token.Channel)

diff --git a/src/sly/parser/generator/EBNFParserBuilder.cs b/src/sly/parser/generator/EBNFParserBuilder.cs
@@ -167,7 +167,10 @@ protected virtual ParserConfiguration<IN, OUT> ExtractEbnfParserConfiguration(Ty
 
                 var nodeNames = (NodeNameAttribute[])m.GetCustomAttributes(typeof(NodeNameAttribute), true);
                 string nodeName = nodeNames != null && nodeNames.Any() ? nodeNames[0].Name : null;
-
+
+                var subNodeNamesAttributes = (SubNodeNamesAttribute[])m.GetCustomAttributes(typeof(SubNodeNamesAttribute), true);
+                string[] subNodeNames = subNodeNamesAttributes != null && subNodeNamesAttributes.Any() ? subNodeNamesAttributes[0].Names : null;
+
                 foreach (var attr in attributes)
                 {
                     var ruleString = attr.RuleString;
@@ -176,6 +179,7 @@ protected virtual ParserConfiguration<IN, OUT> ExtractEbnfParserConfiguration(Ty
                     {
                         var rule = (Rule<IN>)parseResult.Result;
                         rule.NodeName = nodeName;
+                        rule.SubNodeNames = subNodeNames;
                         rule.RuleString = ruleString;
                         rule.SetVisitor(m);
                         NonTerminal<IN> nonT = null;

diff --git a/src/sly/parser/generator/SubNodeNamesAttribute.cs b/src/sly/parser/generator/SubNodeNamesAttribute.cs
@@ -0,0 +1,14 @@
+using System;
+
+namespace sly.parser.generator;
+
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public class SubNodeNamesAttribute : Attribute
+{
+    public  string[] Names { get;  } = null;
+
+    public SubNodeNamesAttribute(params string[] names)
+    {
+        Names = names;
+    }
+}
diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs
@@ -149,6 +149,18 @@ public override SyntaxParseResult<IN> Parse(IList<Token<IN>> tokens, Rule<IN> ru
                 }
                 else
                 {
+                    if (rule.SubNodeNames != null && rule.SubNodeNames.Length > 0)
+                    {
+                        for (int i = 0; i < Math.Min(rule.SubNodeNames.Length,children.Count); i++)
+                        {
+                            var subNodeName = rule.SubNodeNames[i];
+                            if (subNodeName != null)
+                            {
+                                var child = children[i] as ISyntaxNode<IN>;
+                                child.ForceName(subNodeName);
+                            }
+                        }
+                    }
                     node = new SyntaxNode<IN>( nonTerminalName,  children);
                     node.Name = string.IsNullOrEmpty(rule.NodeName) ? nonTerminalName : rule.NodeName;
                     node.ExpressionAffix = rule.ExpressionAffix;

diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs
@@ -21,6 +21,9 @@ public Rule()
 
         public string NodeName { get; set; } = null;
 
+        public string[] SubNodeNames { get; set; } = null;
+
+
         public bool IsByPassRule { get; set; } = false;
 
         // visitors for operation rules

diff --git a/src/sly/parser/syntax/tree/ISyntaxNode.cs b/src/sly/parser/syntax/tree/ISyntaxNode.cs
@@ -13,6 +13,8 @@ public interface ISyntaxNode<IN> where IN : struct
         string Dump(string tab);
 
         string ToJson(int index = 0);
-
+
+        void ForceName(string name);
+
     }
 }
diff --git a/src/sly/parser/syntax/tree/SyntaxLeaf.cs b/src/sly/parser/syntax/tree/SyntaxLeaf.cs
@@ -27,7 +27,9 @@ public string ToJson(int index = 0)
         {
             return $@"""{index}.{Token.TokenID.ToString()}"" : ""{Token.Value}""";
         }
-
-
+
+        public void ForceName(string name)
+        {
+        }
     }
 }
diff --git a/src/sly/parser/syntax/tree/SyntaxNode.cs b/src/sly/parser/syntax/tree/SyntaxNode.cs
@@ -38,6 +38,11 @@ public SyntaxNode(string name, List<ISyntaxNode<IN>> children = null, MethodInfo
 
         public bool HasByPassNodes { get; set; } = false;
 
+        public void ForceName(string name)
+        {
+            Name = name;
+        }
+
         #region expression syntax nodes
 
         [JsonIgnore]

diff --git a/tests/ParserTests/samples/XmlTests.cs b/tests/ParserTests/samples/XmlTests.cs
@@ -10,7 +10,38 @@ public class XmlTests
 {
 
     [Fact]
-    public void TestXmlParserWithLexerModes()
+    public void TestXmlParserWithLexerModesOk()
+    {
+        ParserBuilder<MinimalXmlLexer, string> builder = new ParserBuilder<MinimalXmlLexer, string>();
+        var xmlparser = new MinimalXmlParser();
+        var r = builder.BuildParser(xmlparser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "document");
+        Check.That(r.IsError).IsFalse();
+        var parser = r.Result;
+        var parsed = parser.Parse(@"
+<?xml version=""1.0""?>
+<!-- starting doc -->
+<root name=""root"">
+    <autoInner name=""autoinner1""/>
+    <inner name=""inner"">
+         <?PI name=""pi""?> 
+        <innerinner name=""innerinner"">
+            inner inner content
+        </innerinner>
+    </inner>
+</root>        
+");
+        Check.That(parsed).IsOkParsing();
+        // var tree = parsed.SyntaxTree;
+        // var graphviz = new GraphVizEBNFSyntaxTreeVisitor<MinimalXmlLexer>();
+        // var root = graphviz.VisitTree(tree);
+        // string graph = graphviz.Graph.Compile();
+        // File.Delete("c:\\tmp\\tree.dot");
+        // File.AppendAllText("c:\\tmp\\tree.dot", graph);
+    }
+
+
+    [Fact]
+    public void TestXmlParserWithLexerModesKo()
     {
         ParserBuilder<MinimalXmlLexer, string> builder = new ParserBuilder<MinimalXmlLexer, string>();
         var xmlparser = new MinimalXmlParser();
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,7 +27,9 @@ public string ToJson(int index = 0) @@
             {
                 return $@"""{index}.{Token.TokenID.ToString()}"" : ""{Token.Value}""";
             }
+            public void ForceName(string name)
+            {
+            }
         }
     }