refactor(parser): improve grammar (#1081)

avoid failling into the `AnyChar` rule when parsing content with the `NormalGroup` entrypoint includes refactoring of the `InlineWord` and `Punctuation` rules to match content faster. Signed-off-by: Xavier Coulon <[email protected]>
bytesparadise · Sep 18, 2022 · 481ef43 · 481ef43
1 parent 4d151e0
commit 481ef43
Show file tree

Hide file tree

Showing 16 changed files with 42,956 additions and 70,307 deletions.
diff --git a/pkg/parser/context.go b/pkg/parser/context.go
@@ -27,7 +27,7 @@ func NewParseContext(config *configuration.Configuration, options ...Option) *Pa
 		GlobalStore(frontMatterKey, true),
 		GlobalStore(documentHeaderKey, true),
 		GlobalStore(usermacrosKey, config.Macros),
-		GlobalStore(enabledSubstitutionsKey, attributeDeclarations()),
+		GlobalStore(enabledSubstitutionsKey, normalSubstitutions()),
 	}
 	opts = append(opts, options...)
 	return &ParseContext{

diff --git a/pkg/parser/document_preprocessing.go b/pkg/parser/document_preprocessing.go
@@ -24,7 +24,7 @@ func Preprocess(source io.Reader, config *configuration.Configuration, opts ...O
 
 func preprocess(ctx *ParseContext, source io.Reader) (string, error) {
 	if log.IsLevelEnabled(log.DebugLevel) {
-		log.Debugf("processing file inclusions in %s with leveloffset=%s", ctx.filename, spew.Sdump(ctx.levelOffsets))
+		log.Debugf("preprocessing file inclusions in %s with leveloffset=%s", ctx.filename, spew.Sdump(ctx.levelOffsets))
 	}
 	b := &builder{
 		enabled: true,
@@ -40,9 +40,9 @@ func preprocess(ctx *ParseContext, source io.Reader) (string, error) {
 			// content of line was not relevant in the context of preparsing (ie, it's a regular line), so let's keep it as-is
 			b.Write(line)
 		} else {
-			if log.IsLevelEnabled(log.DebugLevel) {
-				log.Debugf("checking element of type '%T'", element)
-			}
+			// if log.IsLevelEnabled(log.DebugLevel) {
+			// 	log.Debugf("checking element of type '%T'", element)
+			// }
 			switch e := element.(type) {
 			case *types.AttributeDeclaration:
 				ctx.attributes.set(e.Name, e.Value)

diff --git a/pkg/parser/document_processing_aggregate.go b/pkg/parser/document_processing_aggregate.go
@@ -109,9 +109,9 @@ func aggregate(ctx *ParseContext, fragmentStream <-chan types.DocumentFragment)
 }
 
 func resolveCrossReferences(element interface{}, attrs *contextAttributes) error {
-	if log.IsLevelEnabled(log.DebugLevel) {
-		log.Debugf("resolving cross references in element of type '%T'", element)
-	}
+	// if log.IsLevelEnabled(log.DebugLevel) {
+	// 	log.Debugf("resolving cross references in element of type '%T'", element)
+	// }
 	switch e := element.(type) {
 	case types.WithElements:
 		for _, elmt := range e.GetElements() {

diff --git a/pkg/parser/document_processing_apply_substitutions.go b/pkg/parser/document_processing_apply_substitutions.go
@@ -3,6 +3,7 @@ package parser
 import (
 	"bytes"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -29,7 +30,7 @@ func ApplySubstitutions(ctx *ParseContext, done <-chan interface{}, fragmentStre
 				return
 			}
 		}
-		log.WithField("pipeline_stage", "apply_substitutions").Debug("done")
+		log.WithField("pipeline_stage", "apply_substitutions").Info("done")
 	}()
 	return processedFragmentStream
 }
@@ -614,7 +615,8 @@ func parseWithSubstitutions(content interface{}, subs *substitutions, opts ...Op
 	if log.IsLevelEnabled(log.DebugLevel) {
 		log.Debugf("parsing '%s' with '%s' substitutions", serialized, subs.toString())
 	}
-	elements, err := parseContent(serialized, append(opts, GlobalStore(enabledSubstitutionsKey, subs))...)
+	stats := Stats{}
+	elements, err := parseContent(serialized, append(opts, GlobalStore(enabledSubstitutionsKey, subs))...) // , Statistics(&stats, "no match"), Debug(true)
 	if err != nil {
 		return nil, err
 	}
@@ -625,6 +627,28 @@ func parseWithSubstitutions(content interface{}, subs *substitutions, opts ...Op
 	if log.IsLevelEnabled(log.DebugLevel) {
 		log.Debugf("parsed content:\n%s", spew.Sdump(elements))
 	}
+	if log.IsLevelEnabled(log.InfoLevel) {
+		log.Infof("parsed '%s' with '%s' substitutions", serialized, subs.toString())
+		log.Infof("stats:")
+		log.Infof(" expr count: %v", stats.ExprCnt)
+		rules := make([]string, 0, len(stats.ChoiceAltCnt))
+		for rule := range stats.ChoiceAltCnt {
+			rules = append(rules, rule)
+		}
+		sort.Strings(rules)
+		for _, rule := range rules {
+			log.Infof("  %s:", rule)
+			matchs := make([]string, 0, len(stats.ChoiceAltCnt[rule]))
+			for match := range stats.ChoiceAltCnt[rule] {
+				matchs = append(matchs, match)
+			}
+			sort.Strings(matchs)
+			for _, match := range matchs {
+				log.Infof("   - case %s: %d", match, stats.ChoiceAltCnt[rule][match])
+
+			}
+		}
+	}
 	return elements, nil
 }
 

diff --git a/pkg/parser/document_substitutions.go b/pkg/parser/document_substitutions.go
@@ -70,16 +70,6 @@ func attributeSubstitutions() *substitutions {
 	}
 }
 
-func attributeDeclarations() *substitutions {
-	return &substitutions{
-		sequence: []string{
-			InlinePassthroughs,
-			AttributeRefs,
-			SpecialCharacters,
-		},
-	}
-}
-
 func noneSubstitutions() *substitutions {
 	return &substitutions{}
 }

diff --git a/pkg/parser/link_test.go b/pkg/parser/link_test.go
@@ -200,7 +200,7 @@ a link to <{example}>.`
 					Expect(ParseDocument(source)).To(MatchDocument(expected))
 				})
 
-				It("with special character in URL", func() {
+				It("with invalid special character in URL", func() {
 					source := `a link to https://example.com>[].`
 					expected := &types.Document{
 						Elements: []interface{}{
@@ -212,11 +212,14 @@ a link to <{example}>.`
 									&types.InlineLink{
 										Location: &types.Location{
 											Scheme: "https://",
-											Path:   "example.com>",
+											Path:   "example.com",
 										},
 									},
+									&types.SpecialCharacter{
+										Name: ">",
+									},
 									&types.StringElement{
-										Content: ".",
+										Content: "[].",
 									},
 								},
 							},
@@ -289,16 +292,7 @@ a link to <{example}>.`
 						&types.Paragraph{
 							Elements: []interface{}{
 								&types.StringElement{
-									Content: "write to ",
-								},
-								&types.InlineLink{
-									Location: &types.Location{
-										Scheme: "mailto:",
-										Path:   "[email protected]",
-									},
-								},
-								&types.StringElement{
-									Content: ".",
+									Content: "write to [email protected].", // local part must not end with `.`
 								},
 							},
 						},

diff --git a/pkg/parser/paragraph_test.go b/pkg/parser/paragraph_test.go
@@ -345,6 +345,33 @@ and another one using attribute substitution: {github-url}[{github-title}]...
 
 		Context("regular paragraphs", func() {
 
+			It("2 inlinewords and a punctuation", func() {
+				source := "\"`Get moving!`\" he shouted."
+				expected := &types.Document{
+					Elements: []interface{}{
+						&types.Paragraph{
+							Elements: []interface{}{
+								&types.Symbol{
+									Name: "\"`",
+								},
+								&types.StringElement{
+									Content: "Get moving!",
+								},
+								&types.Symbol{
+									Name: "`\"",
+								},
+								&types.StringElement{
+									Content: " he shouted.",
+								},
+							},
+						},
+					},
+				}
+				result, err := ParseDocument(source)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(result).To(MatchDocument(expected))
+			})
+
 			It("3 with basic content", func() {
 				source := `cookie