feature(parser): custom subs on paragraphs

Support for one or more custom substitutions Starting with Rawline elements which are parsed and result in other elements. Subsequent substitutions only apply on the StringElements. At the end of the substitutions, the lines are preserved Also, fix issues with single line comments which should not start with spaces before the `//` marker Fixes #597 Signed-off-by: Xavier Coulon <[email protected]>
bytesparadise · Jul 20, 2020 · dd5d31b · dd5d31b
1 parent 4824d20
commit dd5d31b
Show file tree

Hide file tree

Showing 10 changed files with 5,819 additions and 4,521 deletions.
diff --git a/make/go.mk b/make/go.mk
@@ -42,12 +42,12 @@ generate: install-pigeon
 generate-optimized: install-pigeon
 	@echo "generating the parser (optimized)..."
 	@pigeon -optimize-parser \
-		-alternate-entrypoints AsciidocRawDocument,RawFile,TextDocument,DocumentRawBlock,FileLocation,IncludedFileLine,InlineLinks,LabeledListItemTerm,NormalBlockContent,NormalParagraphContent,VerseBlockContent,MarkdownQuoteBlockAttribution,InlineElements \
+		-alternate-entrypoints AsciidocRawDocument,RawFile,TextDocument,DocumentRawBlock,FileLocation,IncludedFileLine,InlineLinks,LabeledListItemTerm,NormalParagraphContentSubstitution,NormalBlockContentSubstitution,VerseBlockContentSubstitution,MarkdownQuoteBlockAttribution,InlineElements,QuotedTextSubstitution,NoneSubstitution,AttributesSubstitution \
 		-o ./pkg/parser/parser.go ./pkg/parser/parser.peg
 
 .PHONY: build
 ## build the binary executable from CLI
-build: prebuild-checks generate-optimized
+build: prebuild-checks verify-parser generate-optimized
 	$(eval BUILD_COMMIT:=$(shell git rev-parse --short HEAD))
 	$(eval BUILD_TAG:=$(shell git tag --contains $(BUILD_COMMIT)))
 	$(eval BUILD_TIME:=$(shell date -u '+%Y-%m-%dT%H:%M:%SZ'))

diff --git a/pkg/parser/comment_test.go b/pkg/parser/comment_test.go
@@ -26,30 +26,6 @@ var _ = Describe("comments", func() {
 				Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
 			})
 
-			It("single line comment with prefixing spaces alone", func() {
-				source := `  // A single-line comment.`
-				expected := types.DraftDocument{
-					Blocks: []interface{}{
-						types.SingleLineComment{
-							Content: " A single-line comment.",
-						},
-					},
-				}
-				Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
-			})
-
-			It("single line comment with prefixing tabs alone", func() {
-				source := "\t\t// A single-line comment."
-				expected := types.DraftDocument{
-					Blocks: []interface{}{
-						types.SingleLineComment{
-							Content: " A single-line comment.",
-						},
-					},
-				}
-				Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
-			})
-
 			It("single line comment at end of line", func() {
 				source := `foo // A single-line comment.`
 				expected := types.DraftDocument{
@@ -71,7 +47,7 @@ var _ = Describe("comments", func() {
 			It("single line comment within a paragraph", func() {
 				source := `a first line
 // A single-line comment.
-another line`
+another line // with a comment`
 				expected := types.DraftDocument{
 					Blocks: []interface{}{
 						types.Paragraph{
@@ -88,44 +64,85 @@ another line`
 								},
 								[]interface{}{
 									types.StringElement{
-										Content: "another line",
+										Content: "another line // with a comment",
 									},
 								},
 							},
 						},
 					},
 				}
-				Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
+				result, err := ParseDraftDocument(source)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(result).To(MatchDraftDocument(expected))
 			})
 
-			It("single line comment within a paragraph with tab", func() {
-				source := `a first line
+			Context("invalid", func() {
+
+				It("single line comment with prefixing spaces alone", func() {
+					source := `  // A single-line comment.`
+					expected := types.DraftDocument{
+						Blocks: []interface{}{
+							types.LiteralBlock{
+								Attributes: types.Attributes{
+									types.AttrKind:             types.Literal,
+									types.AttrLiteralBlockType: types.LiteralBlockWithSpacesOnFirstLine,
+								},
+								Lines: []string{
+									"  // A single-line comment.",
+								},
+							},
+						},
+					}
+					Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
+				})
+
+				It("single line comment with prefixing tabs alone", func() {
+					source := "\t\t// A single-line comment."
+					expected := types.DraftDocument{
+						Blocks: []interface{}{
+							types.LiteralBlock{
+								Attributes: types.Attributes{
+									types.AttrKind:             types.Literal,
+									types.AttrLiteralBlockType: types.LiteralBlockWithSpacesOnFirstLine,
+								},
+								Lines: []string{
+									"\t\t// A single-line comment.",
+								},
+							},
+						},
+					}
+					Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
+				})
+
+				It("single line comment within a paragraph with tab", func() {
+					source := `a first line
 	// A single-line comment.
 another line`
-				expected := types.DraftDocument{
-					Blocks: []interface{}{
-						types.Paragraph{
-							Lines: []interface{}{
-								[]interface{}{
-									types.StringElement{
-										Content: "a first line",
+					expected := types.DraftDocument{
+						Blocks: []interface{}{
+							types.Paragraph{
+								Lines: []interface{}{
+									[]interface{}{
+										types.StringElement{
+											Content: "a first line",
+										},
 									},
-								},
-								[]interface{}{
-									types.SingleLineComment{
-										Content: " A single-line comment.",
+									[]interface{}{
+										types.StringElement{
+											Content: "\t// A single-line comment.",
+										},
 									},
-								},
-								[]interface{}{
-									types.StringElement{
-										Content: "another line",
+									[]interface{}{
+										types.StringElement{
+											Content: "another line",
+										},
 									},
 								},
 							},
 						},
-					},
-				}
-				Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
+					}
+					Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
+				})
 			})
 		})
 
@@ -216,15 +233,35 @@ a second paragraph`
 			It("single line comment with prefixing spaces alone", func() {
 				source := `  // A single-line comment.`
 				expected := types.Document{
-					Elements: []interface{}{},
+					Elements: []interface{}{
+						types.LiteralBlock{
+							Attributes: types.Attributes{
+								types.AttrKind:             types.Literal,
+								types.AttrLiteralBlockType: types.LiteralBlockWithSpacesOnFirstLine,
+							},
+							Lines: []string{
+								"  // A single-line comment.",
+							},
+						},
+					},
 				}
 				Expect(ParseDocument(source)).To(MatchDocument(expected))
 			})
 
 			It("single line comment with prefixing tabs alone", func() {
 				source := "\t\t// A single-line comment."
 				expected := types.Document{
-					Elements: []interface{}{},
+					Elements: []interface{}{
+						types.LiteralBlock{
+							Attributes: types.Attributes{
+								types.AttrKind:             types.Literal,
+								types.AttrLiteralBlockType: types.LiteralBlockWithSpacesOnFirstLine,
+							},
+							Lines: []string{
+								"\t\t// A single-line comment.",
+							},
+						},
+					},
 				}
 				Expect(ParseDocument(source)).To(MatchDocument(expected))
 			})
@@ -266,25 +303,31 @@ another line`
 				Expect(ParseDocument(source)).To(MatchDocument(expected))
 			})
 
-			It("single line comment within a paragraph with tab", func() {
-				source := `a first line
+			Context("invalid", func() {
+
+				It("single line comment within a paragraph with tab", func() {
+					source := `a first line
 	// A single-line comment.
 another line`
-				expected := types.Document{
-					Elements: []interface{}{
-						types.Paragraph{
-							Lines: []interface{}{
-								[]interface{}{
-									types.StringElement{Content: "a first line"},
-								},
-								[]interface{}{
-									types.StringElement{Content: "another line"},
+					expected := types.Document{
+						Elements: []interface{}{
+							types.Paragraph{
+								Lines: []interface{}{
+									[]interface{}{
+										types.StringElement{Content: "a first line"},
+									},
+									[]interface{}{
+										types.StringElement{Content: "\t// A single-line comment."},
+									},
+									[]interface{}{
+										types.StringElement{Content: "another line"},
+									},
 								},
 							},
 						},
-					},
-				}
-				Expect(ParseDocument(source)).To(MatchDocument(expected))
+					}
+					Expect(ParseDocument(source)).To(MatchDocument(expected))
+				})
 			})
 		})
 

diff --git a/pkg/parser/delimited_block_test.go b/pkg/parser/delimited_block_test.go
@@ -232,6 +232,39 @@ var _ = Describe("delimited blocks", func() {
 				}
 				Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
 			})
+
+			Context("with custom substitutions", func() {
+
+				It("should apply the 'none' substitution", func() {
+					source := "[subs=none]\n```" + "\n" +
+						"a http://website.com[]" + "\n" +
+						"and more text on the" + "\n" +
+						"next lines" + "\n" +
+						"```"
+					expected := types.DraftDocument{
+						Blocks: []interface{}{
+							types.DelimitedBlock{
+								Kind: types.Fenced,
+								Attributes: types.Attributes{
+									types.AttrSubstitutions: "none",
+								},
+								Elements: []interface{}{
+									types.VerbatimLine{
+										Content: "a http://website.com[]",
+									},
+									types.VerbatimLine{
+										Content: "and more text on the",
+									},
+									types.VerbatimLine{
+										Content: "next lines",
+									},
+								},
+							},
+						},
+					}
+					Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
+				})
+			})
 		})
 
 		Context("listing block", func() {
@@ -663,6 +696,10 @@ import <a>
 				}
 				Expect(ParseDraftDocument(source)).To(MatchDraftDocument(expected))
 			})
+
+			Context("with custom substitutions", func() {
+
+			})
 		})
 
 		Context("example block", func() {

diff --git a/pkg/parser/document_processing.go b/pkg/parser/document_processing.go
@@ -73,7 +73,7 @@ func ParseRawDocument(r io.Reader, config configuration.Configuration, options .
 		return types.RawDocument{}, err
 	}
 	if log.IsLevelEnabled(log.DebugLevel) {
-		log.Debug("draft document:")
+		log.Debug("raw document:")
 		spew.Fdump(log.StandardLogger().Out, doc)
 	}
 	return doc, nil