Merge branch 'dev' into project/aot-trimming/building

* dev:
  Bump Microsoft.NET.Test.Sdk from 17.11.0 to 17.11.1
  .
  .
  v3.2.9
  benchmarking
  alpha release
  Bump Microsoft.NET.Test.Sdk from 17.10.0 to 17.11.0
  .
  use generic lexer instead of regex lexer
  use generic lexer instead of regex lexer
  limit lambdas on Rule.Match
  remove useless List<UnexpectedTokenError> creation

src/bench2.4/GenericLexerBench.cs

@@ -22,7 +22,7 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.Current.Value);
+                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp80);
                 Add(baseJob.WithNuGet("sly", "2.4.0").WithId("2.4.0"));
                 Add(EnvironmentAnalyser.Default);
             }

src/bench2.4/JsonParserBench.cs

@@ -31,10 +31,9 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.Current.Value);
-                Add(baseJob.WithNuGet("sly", "2.2.5.1").WithId("2.2.5.1"));
-                Add(baseJob.WithNuGet("sly", "2.3.0").WithId("2.3.0"));
-                Add(baseJob.WithNuGet("sly", "2.4.0.1").WithId("2.4.0.1"));
+                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp80);
+                Add(baseJob.WithNuGet("sly", "3.2.9-alpha1").WithId("3.2.9-alpha1"));
+                Add(baseJob.WithNuGet("sly", "3.2.8").WithId("3.2.8"));
                 Add(EnvironmentAnalyser.Default);
 
             }
@@ -60,6 +59,7 @@ public void Setup()
             {
                 Console.WriteLine("ERROR");
                 result.Errors.ForEach(e => Console.WriteLine(e.Message));
+                Environment.Exit(1);
             }
             else
             {

src/bench2.4/Program.cs

@@ -1,5 +1,8 @@
 using System;
+using bench.json;
+using bench.json.model;
 using BenchmarkDotNet.Running;
+using sly.parser.generator;
 
 namespace bench
 {
@@ -15,6 +18,21 @@ static void Main(string[] args)
         {
             try
             {
+
+                // var builder = new ParserBuilder<JsonTokenGeneric, JSon>();
+                // var jsonParser = new EbnfJsonGenericParser();
+                // var result = builder.BuildParser(jsonParser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
+                // Console.WriteLine("parser built.");
+                // if (result.IsError)
+                // {
+                //     Console.WriteLine("ERROR");
+                //     result.Errors.ForEach(e => Console.WriteLine(e.Message));
+                //     Environment.Exit(1);
+                // }
+                // else
+                // {
+                //     Console.WriteLine("parser ok");
+                // }
 
 
                 Console.WriteLine("Hello World!");

src/bench2.4/bench2.4.csproj

@@ -6,8 +6,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
-    <PackageReference Include="sly" Version="3.0.1" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
+    <PackageReference Include="sly" Version="3.2.8" />
     <PackageReference Include="System.Memory" Version="4.5.5" />    
   </ItemGroup>
 

src/bench2.4/json/EbnfJsonGenericParser.cs

@@ -27,13 +27,13 @@ public JSon StringValue(Token<JsonTokenGeneric> stringToken)
         }
 
         [Production("value : INT")]
-        public object IntValue(Token<JsonTokenGeneric> intToken)
+        public JSon IntValue(Token<JsonTokenGeneric> intToken)
         {
             return new JValue(intToken.IntValue);
         }
 
         [Production("value : DOUBLE")]
-        public object DoubleValue(Token<JsonTokenGeneric> doubleToken)
+        public JSon DoubleValue(Token<JsonTokenGeneric> doubleToken)
         {
             double dbl;
             try
@@ -56,13 +56,13 @@ public object DoubleValue(Token<JsonTokenGeneric> doubleToken)
         }
 
         [Production("value : BOOLEAN")]
-        public object BooleanValue(Token<JsonTokenGeneric> boolToken)
+        public JSon BooleanValue(Token<JsonTokenGeneric> boolToken)
         {
             return new JValue(bool.Parse(boolToken.Value));
         }
 
-        [Production("value : NULL")]
-        public object NullValue(object forget)
+        [Production("value : NULL[d]")]
+        public JSon NullValue()
         {
             return new JNull();
         }
@@ -83,14 +83,14 @@ public JSon ListValue(JList list)
 
         #region OBJECT
 
-        [Production("object: ACCG ACCD")]
-        public JSon EmptyObjectValue(object accg, object accd)
+        [Production("object: ACCG[d] ACCD[d]")]
+        public JSon EmptyObjectValue()
         {
             return new JObject();
         }
 
-        [Production("object: ACCG members ACCD")]
-        public JSon AttributesObjectValue(object accg, JObject members, object accd)
+        [Production("object: ACCG[d] members ACCD[d]")]
+        public JSon AttributesObjectValue(JObject members)
         {
             return members;
         }
@@ -99,14 +99,14 @@ public JSon AttributesObjectValue(object accg, JObject members, object accd)
 
         #region LIST
 
-        [Production("list: CROG CROD")]
-        public JSon EmptyList(object crog, object crod)
+        [Production("list: CROG[d] CROD[d]")]
+        public JSon EmptyList()
         {
             return new JList();
         }
 
-        [Production("list: CROG listElements CROD")]
-        public JSon List(object crog, JList elements, object crod)
+        [Production("list: CROG[d] listElements CROD[d]")]
+        public JSon List(JList elements)
         {
             return elements;
         }
@@ -131,7 +131,7 @@ public JSon ListElementsOne(Token<JsonTokenGeneric> discardedComma, JSon value)
         #region PROPERTIES
 
         [Production("members: property additionalProperty*")]
-        public object Members(JObject head, List<JSon> tail)
+        public JSon Members(JObject head, List<JSon> tail)
         {
             var value = new JObject();
             value.Merge(head);
@@ -140,13 +140,13 @@ public object Members(JObject head, List<JSon> tail)
         }
 
         [Production("additionalProperty : COMMA property")]
-        public object property(Token<JsonTokenGeneric> comma, JObject property)
+        public JSon property(Token<JsonTokenGeneric> comma, JObject property)
         {
             return property;
         }
 
-        [Production("property: STRING COLON value")]
-        public object property(Token<JsonTokenGeneric> key, object colon, JSon value)
+        [Production("property: STRING COLON[d] value")]
+        public JSon property(Token<JsonTokenGeneric> key, JSon value)
         {
             return new JObject(key.StringWithoutQuotes, value);
         }

src/benchCurrent/Program.cs

@@ -8,11 +8,13 @@ static class Program
 
         private static void BenchJson() {
 
-            var summary = BenchmarkRunner.Run<JsonParserBench>();
+            // var summary = BenchmarkRunner.Run<JsonParserBench>();
+            //
+            // var summary2 = BenchmarkRunner.Run<BackTrackBench>();
 
-            var summary2 = BenchmarkRunner.Run<BackTrackBench>();
-            
-            var summary3 = BenchmarkRunner.Run<WhileBench>();
+            // var summary3 = BenchmarkRunner.Run<WhileBench>();
+
+            var summary4 = BenchmarkRunner.Run<SimpleExpressionBench>();
 
         }
         static void Main(string[] args)

src/benchCurrent/SimpleExpressionBench.cs

@@ -0,0 +1,114 @@
+using System;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains.CsProj;
+using expressionparser;
+using simpleExpressionParser;
+using sly.parser;
+using sly.parser.generator;
+using ExpressionToken = simpleExpressionParser.ExpressionToken;
+
+namespace benchCurrent
+{
+
+    [MemoryDiagnoser]
+
+    [Config(typeof(Config))]
+    public class SimpleExpressionBench
+    {
+
+
+        private class Config : ManualConfig
+        {
+            public Config()
+            {
+                var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp70);
+            }
+        }
+
+        private Parser<ExpressionToken, double> BenchedParser;
+        private Parser<GenericExpressionToken, double> BenchedGenericParser;
+
+        private string content = "";
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            Console.WriteLine(("SETUP"));
+            content = "1+2+3+4+5+6+7+8+9+10+11+12+13+14+15+16+17+18+19+20";
+
+            SimpleExpressionParser p = new SimpleExpressionParser();
+            var builder = new ParserBuilder<ExpressionToken, double>();
+
+            var result = builder.BuildParser(p, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
+
+            if (result.IsError)
+            {
+                result.Errors.ForEach(x => Console.WriteLine(x.Message));
+                Environment.Exit(1);
+            }
+            else
+            {
+                Console.WriteLine("parser ok");
+                BenchedParser = result.Result;
+            }
+
+            GenericSimpleExpressionParser gp = new GenericSimpleExpressionParser();
+            var genericbuilder = new ParserBuilder<GenericExpressionToken, double>();
+
+            var genericresult = genericbuilder.BuildParser(gp, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
+
+            if (genericresult.IsError)
+            {
+                genericresult.Errors.ForEach(x => Console.WriteLine(x.Message));
+                Environment.Exit(2);
+            }
+            else
+            {
+                Console.WriteLine("parser ok");
+                BenchedGenericParser = genericresult.Result;
+            }
+        }
+
+
+
+        [Benchmark]
+
+        public void TestExpressionRegex()
+        {
+            if (BenchedParser == null)
+            {
+                Console.WriteLine("regex parser is null");
+            }
+            else
+            {
+                for (int i = 0; i < 50; i++)
+                {
+                    var ignored = BenchedParser.Parse(content);
+                }
+            }
+        }
+
+        // [Benchmark]
+
+        public void TestExpressionGeneric()
+        {
+            if (BenchedGenericParser == null)
+            {
+                Console.WriteLine("generic parser is null");
+            }
+            else
+            {
+                for (int i = 0; i < 50; i++)
+                {
+                    var ignored = BenchedGenericParser.Parse(content);
+                }
+            }
+        }
+
+
+
+    }
+
+}

src/benchCurrent/benchCurrent.csproj

@@ -27,7 +27,9 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\samples\expressionParser\expressionParser.csproj" />
     <ProjectReference Include="..\samples\IndentedWhile\indentedWhile.csproj" />
+    <ProjectReference Include="..\samples\SimpleExpressionParser\SimpleExpressionParser.csproj" />
     <ProjectReference Include="..\sly\sly.csproj" />
   </ItemGroup>
 

src/samples/ParserExample/Program.cs

@@ -38,6 +38,7 @@
 using XML;
 using Xunit;
 using ExpressionContext = postProcessedLexerParser.expressionModel.ExpressionContext;
+using ExpressionToken = simpleExpressionParser.ExpressionToken;
 using IfThenElse = indented.IfThenElse;
 
 namespace ParserExample
@@ -131,6 +132,26 @@ public static object Rec(List<object> args)
         }
 
 
+        private static void BenchSimpleExpression()
+        {
+            GenericSimpleExpressionParser p = new GenericSimpleExpressionParser();
+            var builder = new ParserBuilder<GenericExpressionToken, double>();
+
+            var Parser = builder.BuildParser(p, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
+            if (Parser.IsOk)
+            {
+                for (int i = 0; i < 50; i++)
+                {
+                    var r = Parser.Result.Parse("1+2+3+4+5+6+7+8+9+10+11+12+13+14+15+16+17+18+19+20");
+                    if (r.IsOk)
+                    {
+                        Console.WriteLine(r.Result);
+                    }    
+                }
+
+            }
+        }
+
         private static void TestFactorial()
         {
             var whileParser = new WhileParserGeneric();
@@ -1164,7 +1185,8 @@ print a
         }
         private static void Main(string[] args)
         {
-            IndentRefactoring();
+            BenchSimpleExpression();
+            // IndentRefactoring();
             //NodeNames();
             // BroadWindow();
             // return;