Skip to content

Commit

Permalink
sub node names attributes
Browse files Browse the repository at this point in the history
  • Loading branch information
b3b00 committed Dec 18, 2024
1 parent 3ec044a commit 72ed3b9
Show file tree
Hide file tree
Showing 12 changed files with 154 additions and 26 deletions.
25 changes: 10 additions & 15 deletions src/samples/XML/MinimalXmlParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,16 @@ public string Document(List<string> startingMiscs, string root, List<string> end
return b.ToString();
}

[Production("element : OPEN[d] ID attributes SLASH[d] CLOSE[d]")]
public string AutoElement(Token<MinimalXmlLexer> id, string attributes)
[Production("element : OPEN[d] ID attribute* SLASH[d] CLOSE[d]")]
public string AutoElement(Token<MinimalXmlLexer> id, List<string> attributes)
{
return $"autoTag({id.Value}, {attributes})";
return $"autoTag({id.Value}, {string.Join(", ",attributes.Select(x => x.ToString()))})";
}

[Production("opentag : OPEN[d] ID attributes CLOSE[d]")]
public string OpenTag(Token<MinimalXmlLexer> tagName, string attributes)
[Production("opentag : OPEN[d] ID attribute* CLOSE[d]")]
public string OpenTag(Token<MinimalXmlLexer> tagName, List<string> attributes)
{
return $"open ({tagName.Value}, {attributes})";
return $"open ({tagName.Value}, {string.Join(", ",attributes.Select(x => x.ToString()))})";
}

[Production("closetag : OPEN[d] SLASH[d] ID CLOSE[d]")]
Expand All @@ -48,6 +48,7 @@ public string CloseTag(Token<MinimalXmlLexer> id)
return $"close({id.Value})";
}

[SubNodeNames(null, "elements",null)]
[Production("element : opentag [element|pi|comment|content]* closetag")]
public string CompoundElement(string open, List<string> subs, string close)
{
Expand All @@ -67,18 +68,12 @@ public string Comment(Token<MinimalXmlLexer> comment)
return $"comment({comment.Value})";
}

[Production("pi : OPEN_PI[d] ID attributes CLOSE_PI[d]")]
public string Pi(Token<MinimalXmlLexer> id , string attributes)
[Production("pi : OPEN_PI[d] ID attribute* CLOSE_PI[d]")]
public string Pi(Token<MinimalXmlLexer> id , List<string> attributes)
{
return $"pi({id.Value} :: {attributes.ToString()})";
return $"pi({id.Value} :: {string.Join(", ",attributes.Select(x => x.ToString()))})";
}


[Production("attributes : attribute*")]
public string Attributes(List<string> attributes)
{
return string.Join(", ",attributes.Select(x => x.ToString()));
}

[Production("attribute: ID EQUALS[d] VALUE")]
public string Attribute(Token<MinimalXmlLexer> id, Token<MinimalXmlLexer> value)
Expand Down
62 changes: 62 additions & 0 deletions src/samples/XML/xml.grammar
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@

genericLexer MinimalXmlLexer;

[Mode]
[Push("tag")]
[Sugar] OPEN : "<";

[UpTo] CONTENT : "<";

[Mode]
[Push("pi")]
[Sugar] OPEN_PI : "<?";

[Mode]
[MultiLineComment] COMMENT : "<!--" "-->";

[Mode("tag")]
[Mode("pi")]
[AlphaId] ID;

[Mode("tag")]
[Sugar] SLASH : "/";

[Mode("tag", "pi")]
[Sugar] EQUALS : "=";

[Mode("tag", "pi")]
[Mode("pi")]
[String] VALUE;

[Mode("pi")]
[Pop]
[Sugar] CLOSE_PI : "?>";

[Mode("tag")]
[Pop]
[Sugar] CLOSE : ">";



parser MinimalXmlParser;

-> document : misc* element misc*;

element : OPEN[d] ID attribute* SLASH[d] CLOSE[d];

opentag : OPEN[d] ID attribute* CLOSE[d];

closetag : OPEN[d] SLASH[d] ID CLOSE[d];

element : opentag [element|pi|comment|content]* closetag;

misc : [comment | pi | content];

comment : COMMENT;

pi : OPEN_PI[d] ID attribute* CLOSE_PI[d];

attribute: ID EQUALS[d] VALUE;

content : CONTENT;

6 changes: 2 additions & 4 deletions src/sly/lexer/TokenChannel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,9 @@ public Token<IN> this[int key]
set => SetToken(key,value);
}

public void Add(Token<IN> token)
public void Shift()
{
Tokens.Add(token);
if (token != null)
token.PositionInTokenFlow = Tokens.Count;
Tokens.Add(null);
}

[ExcludeFromCodeCoverage]
Expand Down
4 changes: 2 additions & 2 deletions src/sly/lexer/TokenChannels.cs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void Add(Token<IN> token)
}
for (int i = 0; i < shift; i++)
{
channel.Add(null);
channel.Shift();
}
_tokenChannels[token.Channel] = channel;
}
Expand All @@ -92,7 +92,7 @@ public void Add(Token<IN> token)
{
for (int i = channel.Count; i < index; i++)
{
channel.Add(null);
channel.Shift();
}

if (channel.ChannelId == token.Channel)
Expand Down
6 changes: 5 additions & 1 deletion src/sly/parser/generator/EBNFParserBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,10 @@ protected virtual ParserConfiguration<IN, OUT> ExtractEbnfParserConfiguration(Ty

var nodeNames = (NodeNameAttribute[])m.GetCustomAttributes(typeof(NodeNameAttribute), true);
string nodeName = nodeNames != null && nodeNames.Any() ? nodeNames[0].Name : null;


var subNodeNamesAttributes = (SubNodeNamesAttribute[])m.GetCustomAttributes(typeof(SubNodeNamesAttribute), true);
string[] subNodeNames = subNodeNamesAttributes != null && subNodeNamesAttributes.Any() ? subNodeNamesAttributes[0].Names : null;

foreach (var attr in attributes)
{
var ruleString = attr.RuleString;
Expand All @@ -176,6 +179,7 @@ protected virtual ParserConfiguration<IN, OUT> ExtractEbnfParserConfiguration(Ty
{
var rule = (Rule<IN>)parseResult.Result;
rule.NodeName = nodeName;
rule.SubNodeNames = subNodeNames;
rule.RuleString = ruleString;
rule.SetVisitor(m);
NonTerminal<IN> nonT = null;
Expand Down
14 changes: 14 additions & 0 deletions src/sly/parser/generator/SubNodeNamesAttribute.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
using System;

namespace sly.parser.generator;

[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class SubNodeNamesAttribute : Attribute
{
public string[] Names { get; } = null;

public SubNodeNamesAttribute(params string[] names)
{
Names = names;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,18 @@ public override SyntaxParseResult<IN> Parse(IList<Token<IN>> tokens, Rule<IN> ru
}
else
{
if (rule.SubNodeNames != null && rule.SubNodeNames.Length > 0)
{
for (int i = 0; i < Math.Min(rule.SubNodeNames.Length,children.Count); i++)
{
var subNodeName = rule.SubNodeNames[i];
if (subNodeName != null)
{
var child = children[i] as ISyntaxNode<IN>;
child.ForceName(subNodeName);
}
}
}
node = new SyntaxNode<IN>( nonTerminalName, children);
node.Name = string.IsNullOrEmpty(rule.NodeName) ? nonTerminalName : rule.NodeName;
node.ExpressionAffix = rule.ExpressionAffix;
Expand Down
3 changes: 3 additions & 0 deletions src/sly/parser/syntax/grammar/Rule.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ public Rule()

public string NodeName { get; set; } = null;

public string[] SubNodeNames { get; set; } = null;


public bool IsByPassRule { get; set; } = false;

// visitors for operation rules
Expand Down
4 changes: 3 additions & 1 deletion src/sly/parser/syntax/tree/ISyntaxNode.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public interface ISyntaxNode<IN> where IN : struct
string Dump(string tab);

string ToJson(int index = 0);


void ForceName(string name);

}
}
6 changes: 4 additions & 2 deletions src/sly/parser/syntax/tree/SyntaxLeaf.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ public string ToJson(int index = 0)
{
return $@"""{index}.{Token.TokenID.ToString()}"" : ""{Token.Value}""";
}



public void ForceName(string name)
{
}
}
}
5 changes: 5 additions & 0 deletions src/sly/parser/syntax/tree/SyntaxNode.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ public SyntaxNode(string name, List<ISyntaxNode<IN>> children = null, MethodInfo

public bool HasByPassNodes { get; set; } = false;

public void ForceName(string name)
{
Name = name;
}

#region expression syntax nodes

[JsonIgnore]
Expand Down
33 changes: 32 additions & 1 deletion tests/ParserTests/samples/XmlTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,38 @@ public class XmlTests
{

[Fact]
public void TestXmlParserWithLexerModes()
public void TestXmlParserWithLexerModesOk()
{
ParserBuilder<MinimalXmlLexer, string> builder = new ParserBuilder<MinimalXmlLexer, string>();
var xmlparser = new MinimalXmlParser();
var r = builder.BuildParser(xmlparser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "document");
Check.That(r.IsError).IsFalse();
var parser = r.Result;
var parsed = parser.Parse(@"
<?xml version=""1.0""?>
<!-- starting doc -->
<root name=""root"">
<autoInner name=""autoinner1""/>
<inner name=""inner"">
<?PI name=""pi""?>
<innerinner name=""innerinner"">
inner inner content
</innerinner>
</inner>
</root>
");
Check.That(parsed).IsOkParsing();
// var tree = parsed.SyntaxTree;
// var graphviz = new GraphVizEBNFSyntaxTreeVisitor<MinimalXmlLexer>();
// var root = graphviz.VisitTree(tree);
// string graph = graphviz.Graph.Compile();
// File.Delete("c:\\tmp\\tree.dot");
// File.AppendAllText("c:\\tmp\\tree.dot", graph);
}


[Fact]
public void TestXmlParserWithLexerModesKo()
{
ParserBuilder<MinimalXmlLexer, string> builder = new ParserBuilder<MinimalXmlLexer, string>();
var xmlparser = new MinimalXmlParser();
Expand Down

0 comments on commit 72ed3b9

Please sign in to comment.