From 72ed3b91170dd677160c4c422e40a7f1e554c1c5 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 18 Dec 2024 18:00:40 +0100 Subject: [PATCH] sub node names attributes --- src/samples/XML/MinimalXmlParser.cs | 25 +++----- src/samples/XML/xml.grammar | 62 +++++++++++++++++++ src/sly/lexer/TokenChannel.cs | 6 +- src/sly/lexer/TokenChannels.cs | 4 +- src/sly/parser/generator/EBNFParserBuilder.cs | 6 +- .../parser/generator/SubNodeNamesAttribute.cs | 14 +++++ .../ebnf/EBNFRecursiveDescentSyntaxParser.cs | 12 ++++ src/sly/parser/syntax/grammar/Rule.cs | 3 + src/sly/parser/syntax/tree/ISyntaxNode.cs | 4 +- src/sly/parser/syntax/tree/SyntaxLeaf.cs | 6 +- src/sly/parser/syntax/tree/SyntaxNode.cs | 5 ++ tests/ParserTests/samples/XmlTests.cs | 33 +++++++++- 12 files changed, 154 insertions(+), 26 deletions(-) create mode 100644 src/samples/XML/xml.grammar create mode 100644 src/sly/parser/generator/SubNodeNamesAttribute.cs diff --git a/src/samples/XML/MinimalXmlParser.cs b/src/samples/XML/MinimalXmlParser.cs index a129b412..b1198f0e 100644 --- a/src/samples/XML/MinimalXmlParser.cs +++ b/src/samples/XML/MinimalXmlParser.cs @@ -30,16 +30,16 @@ public string Document(List startingMiscs, string root, List end return b.ToString(); } - [Production("element : OPEN[d] ID attributes SLASH[d] CLOSE[d]")] - public string AutoElement(Token id, string attributes) + [Production("element : OPEN[d] ID attribute* SLASH[d] CLOSE[d]")] + public string AutoElement(Token id, List attributes) { - return $"autoTag({id.Value}, {attributes})"; + return $"autoTag({id.Value}, {string.Join(", ",attributes.Select(x => x.ToString()))})"; } - [Production("opentag : OPEN[d] ID attributes CLOSE[d]")] - public string OpenTag(Token tagName, string attributes) + [Production("opentag : OPEN[d] ID attribute* CLOSE[d]")] + public string OpenTag(Token tagName, List attributes) { - return $"open ({tagName.Value}, {attributes})"; + return $"open ({tagName.Value}, {string.Join(", ",attributes.Select(x => x.ToString()))})"; } [Production("closetag : OPEN[d] SLASH[d] ID CLOSE[d]")] @@ -48,6 +48,7 @@ public string CloseTag(Token id) return $"close({id.Value})"; } + [SubNodeNames(null, "elements",null)] [Production("element : opentag [element|pi|comment|content]* closetag")] public string CompoundElement(string open, List subs, string close) { @@ -67,18 +68,12 @@ public string Comment(Token comment) return $"comment({comment.Value})"; } - [Production("pi : OPEN_PI[d] ID attributes CLOSE_PI[d]")] - public string Pi(Token id , string attributes) + [Production("pi : OPEN_PI[d] ID attribute* CLOSE_PI[d]")] + public string Pi(Token id , List attributes) { - return $"pi({id.Value} :: {attributes.ToString()})"; + return $"pi({id.Value} :: {string.Join(", ",attributes.Select(x => x.ToString()))})"; } - - [Production("attributes : attribute*")] - public string Attributes(List attributes) - { - return string.Join(", ",attributes.Select(x => x.ToString())); - } [Production("attribute: ID EQUALS[d] VALUE")] public string Attribute(Token id, Token value) diff --git a/src/samples/XML/xml.grammar b/src/samples/XML/xml.grammar new file mode 100644 index 00000000..487475cc --- /dev/null +++ b/src/samples/XML/xml.grammar @@ -0,0 +1,62 @@ + +genericLexer MinimalXmlLexer; + +[Mode] +[Push("tag")] +[Sugar] OPEN : "<"; + +[UpTo] CONTENT : "<"; + +[Mode] +[Push("pi")] +[Sugar] OPEN_PI : ""; + +[Mode("tag")] +[Mode("pi")] +[AlphaId] ID; + +[Mode("tag")] +[Sugar] SLASH : "/"; + +[Mode("tag", "pi")] +[Sugar] EQUALS : "="; + +[Mode("tag", "pi")] +[Mode("pi")] +[String] VALUE; + +[Mode("pi")] +[Pop] +[Sugar] CLOSE_PI : "?>"; + +[Mode("tag")] +[Pop] +[Sugar] CLOSE : ">"; + + + +parser MinimalXmlParser; + +-> document : misc* element misc*; + +element : OPEN[d] ID attribute* SLASH[d] CLOSE[d]; + +opentag : OPEN[d] ID attribute* CLOSE[d]; + +closetag : OPEN[d] SLASH[d] ID CLOSE[d]; + +element : opentag [element|pi|comment|content]* closetag; + +misc : [comment | pi | content]; + +comment : COMMENT; + +pi : OPEN_PI[d] ID attribute* CLOSE_PI[d]; + +attribute: ID EQUALS[d] VALUE; + +content : CONTENT; + diff --git a/src/sly/lexer/TokenChannel.cs b/src/sly/lexer/TokenChannel.cs index ed5a3324..28ec380c 100644 --- a/src/sly/lexer/TokenChannel.cs +++ b/src/sly/lexer/TokenChannel.cs @@ -50,11 +50,9 @@ public Token this[int key] set => SetToken(key,value); } - public void Add(Token token) + public void Shift() { - Tokens.Add(token); - if (token != null) - token.PositionInTokenFlow = Tokens.Count; + Tokens.Add(null); } [ExcludeFromCodeCoverage] diff --git a/src/sly/lexer/TokenChannels.cs b/src/sly/lexer/TokenChannels.cs index 316ba1ec..311d599a 100644 --- a/src/sly/lexer/TokenChannels.cs +++ b/src/sly/lexer/TokenChannels.cs @@ -82,7 +82,7 @@ public void Add(Token token) } for (int i = 0; i < shift; i++) { - channel.Add(null); + channel.Shift(); } _tokenChannels[token.Channel] = channel; } @@ -92,7 +92,7 @@ public void Add(Token token) { for (int i = channel.Count; i < index; i++) { - channel.Add(null); + channel.Shift(); } if (channel.ChannelId == token.Channel) diff --git a/src/sly/parser/generator/EBNFParserBuilder.cs b/src/sly/parser/generator/EBNFParserBuilder.cs index 3fd86336..cc457745 100644 --- a/src/sly/parser/generator/EBNFParserBuilder.cs +++ b/src/sly/parser/generator/EBNFParserBuilder.cs @@ -167,7 +167,10 @@ protected virtual ParserConfiguration ExtractEbnfParserConfiguration(Ty var nodeNames = (NodeNameAttribute[])m.GetCustomAttributes(typeof(NodeNameAttribute), true); string nodeName = nodeNames != null && nodeNames.Any() ? nodeNames[0].Name : null; - + + var subNodeNamesAttributes = (SubNodeNamesAttribute[])m.GetCustomAttributes(typeof(SubNodeNamesAttribute), true); + string[] subNodeNames = subNodeNamesAttributes != null && subNodeNamesAttributes.Any() ? subNodeNamesAttributes[0].Names : null; + foreach (var attr in attributes) { var ruleString = attr.RuleString; @@ -176,6 +179,7 @@ protected virtual ParserConfiguration ExtractEbnfParserConfiguration(Ty { var rule = (Rule)parseResult.Result; rule.NodeName = nodeName; + rule.SubNodeNames = subNodeNames; rule.RuleString = ruleString; rule.SetVisitor(m); NonTerminal nonT = null; diff --git a/src/sly/parser/generator/SubNodeNamesAttribute.cs b/src/sly/parser/generator/SubNodeNamesAttribute.cs new file mode 100644 index 00000000..2253e1bc --- /dev/null +++ b/src/sly/parser/generator/SubNodeNamesAttribute.cs @@ -0,0 +1,14 @@ +using System; + +namespace sly.parser.generator; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public class SubNodeNamesAttribute : Attribute +{ + public string[] Names { get; } = null; + + public SubNodeNamesAttribute(params string[] names) + { + Names = names; + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs index 57b0ef26..820f66d2 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs @@ -149,6 +149,18 @@ public override SyntaxParseResult Parse(IList> tokens, Rule ru } else { + if (rule.SubNodeNames != null && rule.SubNodeNames.Length > 0) + { + for (int i = 0; i < Math.Min(rule.SubNodeNames.Length,children.Count); i++) + { + var subNodeName = rule.SubNodeNames[i]; + if (subNodeName != null) + { + var child = children[i] as ISyntaxNode; + child.ForceName(subNodeName); + } + } + } node = new SyntaxNode( nonTerminalName, children); node.Name = string.IsNullOrEmpty(rule.NodeName) ? nonTerminalName : rule.NodeName; node.ExpressionAffix = rule.ExpressionAffix; diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index 7162bef4..4fd114e7 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -21,6 +21,9 @@ public Rule() public string NodeName { get; set; } = null; + public string[] SubNodeNames { get; set; } = null; + + public bool IsByPassRule { get; set; } = false; // visitors for operation rules diff --git a/src/sly/parser/syntax/tree/ISyntaxNode.cs b/src/sly/parser/syntax/tree/ISyntaxNode.cs index 0f65636f..402e5b25 100644 --- a/src/sly/parser/syntax/tree/ISyntaxNode.cs +++ b/src/sly/parser/syntax/tree/ISyntaxNode.cs @@ -13,6 +13,8 @@ public interface ISyntaxNode where IN : struct string Dump(string tab); string ToJson(int index = 0); - + + void ForceName(string name); + } } \ No newline at end of file diff --git a/src/sly/parser/syntax/tree/SyntaxLeaf.cs b/src/sly/parser/syntax/tree/SyntaxLeaf.cs index a4227ac5..98bd4123 100644 --- a/src/sly/parser/syntax/tree/SyntaxLeaf.cs +++ b/src/sly/parser/syntax/tree/SyntaxLeaf.cs @@ -27,7 +27,9 @@ public string ToJson(int index = 0) { return $@"""{index}.{Token.TokenID.ToString()}"" : ""{Token.Value}"""; } - - + + public void ForceName(string name) + { + } } } \ No newline at end of file diff --git a/src/sly/parser/syntax/tree/SyntaxNode.cs b/src/sly/parser/syntax/tree/SyntaxNode.cs index e2c6b2bd..06b2e711 100644 --- a/src/sly/parser/syntax/tree/SyntaxNode.cs +++ b/src/sly/parser/syntax/tree/SyntaxNode.cs @@ -38,6 +38,11 @@ public SyntaxNode(string name, List> children = null, MethodInfo public bool HasByPassNodes { get; set; } = false; + public void ForceName(string name) + { + Name = name; + } + #region expression syntax nodes [JsonIgnore] diff --git a/tests/ParserTests/samples/XmlTests.cs b/tests/ParserTests/samples/XmlTests.cs index 2ea1ac80..3288d0f7 100644 --- a/tests/ParserTests/samples/XmlTests.cs +++ b/tests/ParserTests/samples/XmlTests.cs @@ -10,7 +10,38 @@ public class XmlTests { [Fact] - public void TestXmlParserWithLexerModes() + public void TestXmlParserWithLexerModesOk() + { + ParserBuilder builder = new ParserBuilder(); + var xmlparser = new MinimalXmlParser(); + var r = builder.BuildParser(xmlparser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "document"); + Check.That(r.IsError).IsFalse(); + var parser = r.Result; + var parsed = parser.Parse(@" + + + + + + + + inner inner content + + + +"); + Check.That(parsed).IsOkParsing(); + // var tree = parsed.SyntaxTree; + // var graphviz = new GraphVizEBNFSyntaxTreeVisitor(); + // var root = graphviz.VisitTree(tree); + // string graph = graphviz.Graph.Compile(); + // File.Delete("c:\\tmp\\tree.dot"); + // File.AppendAllText("c:\\tmp\\tree.dot", graph); + } + + + [Fact] + public void TestXmlParserWithLexerModesKo() { ParserBuilder builder = new ParserBuilder(); var xmlparser = new MinimalXmlParser();