fix(renderer): non-ascii characters in listing content

do not replace with HTML entities, which caused troubles with syntax highlighting. also, remove StringElement template (no real value) Fixes #962 Signed-off-by: Xavier Coulon <[email protected]>
bytesparadise · Mar 10, 2022 · 797a610 · 797a610
1 parent 4dcf7c6
commit 797a610
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 19 deletions.
diff --git a/pkg/renderer/sgml/elements.go b/pkg/renderer/sgml/elements.go
@@ -144,7 +144,7 @@ func (r *sgmlRenderer) renderPlainText(ctx *renderer.Context, element interface{
 	case *types.Symbol:
 		return r.renderSymbol(e)
 	case *types.StringElement:
-		return r.renderStringElement(ctx, e)
+		return e.Content, nil
 	case *types.QuotedString:
 		return r.renderQuotedString(ctx, e)
 	// case *types.Paragraph:

diff --git a/pkg/renderer/sgml/html5/delimited_block_source_test.go b/pkg/renderer/sgml/html5/delimited_block_source_test.go
@@ -439,6 +439,42 @@ const Cookie = "cookie"
 <pre class="pygments highlight nowrap"><code data-lang="go"><span style="color:#069;font-weight:bold">const</span> Cookie = <span style="color:#c30">&#34;cookie&#34;</span></code></pre>
 </div>
 </div>
+`
+				Expect(RenderHTML(source)).To(MatchHTML(expected))
+			})
+
+			It("with non-ascii content and syntax highlighting", func() {
+				// source content is `…`, not `...`
+				source := `:source-highlighter: chroma
+:unicode: false
+
+[source,c]
+----
+…
+----`
+				expected := `<div class="listingblock">
+<div class="content">
+<pre class="chroma highlight"><code data-lang="c"><span class="tok-err">…</span></code></pre>
+</div>
+</div>
+`
+				Expect(RenderHTML(source)).To(MatchHTML(expected))
+			})
+
+			It("with symbol content and syntax highlighting", func() {
+				// source content is `…`, not `...`
+				source := `:source-highlighter: chroma
+:unicode: false
+
+[source,c]
+----
+Copyright (C)
+----`
+				expected := `<div class="listingblock">
+<div class="content">
+<pre class="chroma highlight"><code data-lang="c"><span class="tok-n">Copyright</span> <span class="tok-p">(</span><span class="tok-n">C</span><span class="tok-p">)</span></code></pre>
+</div>
+</div>
 `
 				Expect(RenderHTML(source)).To(MatchHTML(expected))
 			})
@@ -447,7 +483,7 @@ const Cookie = "cookie"
 
 	Context("as Markdown block", func() {
 
-		It("with html content", func() {
+		It("with HTML content", func() {
 			source := ".title\n" +
 				"```html\n" +
 				"<!DOCTYPE html>\n" +
@@ -460,6 +496,42 @@ const Cookie = "cookie"
 &lt;/html&gt;</code></pre>
 </div>
 </div>
+`
+			Expect(RenderHTML(source)).To(MatchHTML(expected))
+		})
+
+		It("with non-ascii content and syntax highlighting", func() {
+			// source content is `…`, not `...`, but the `unicode:false` attribute should NOT affect the content of the listing/source block
+			source :=
+				":source-highlighter: chroma\n" +
+					":unicode: false\n" +
+					"\n" +
+					"```c\n" +
+					"…\n" +
+					"```\n"
+			expected := `<div class="listingblock">
+<div class="content">
+<pre class="chroma highlight"><code data-lang="c"><span class="tok-err">…</span></code></pre>
+</div>
+</div>
+`
+			Expect(RenderHTML(source)).To(MatchHTML(expected))
+		})
+
+		It("with symbol content and syntax highlighting", func() {
+			// source content is `…`, not `...`, but the `unicode:false` attribute should NOT affect the content of the listing/source block
+			source :=
+				":source-highlighter: chroma\n" +
+					":unicode: false\n" +
+					"\n" +
+					"```c\n" +
+					"Copyright (C)\n" +
+					"```\n"
+			expected := `<div class="listingblock">
+<div class="content">
+<pre class="chroma highlight"><code data-lang="c"><span class="tok-n">Copyright</span> <span class="tok-p">(</span><span class="tok-n">C</span><span class="tok-p">)</span></code></pre>
+</div>
+</div>
 `
 			Expect(RenderHTML(source)).To(MatchHTML(expected))
 		})

diff --git a/pkg/renderer/sgml/html5/string.go b/pkg/renderer/sgml/html5/string.go
diff --git a/pkg/renderer/sgml/html5/templates.go b/pkg/renderer/sgml/html5/templates.go
@@ -66,7 +66,6 @@ var templates = sgml.Templates{
 	SectionHeader:             sectionHeaderTmpl,
 	SidebarBlock:              sidebarBlockTmpl,
 	SourceBlock:               sourceBlockTmpl,
-	StringElement:             stringTmpl,
 	SubscriptText:             subscriptTextTmpl,
 	SuperscriptText:           superscriptTextTmpl,
 	Table:                     tableTmpl,

diff --git a/pkg/renderer/sgml/sgml_renderer.go b/pkg/renderer/sgml/sgml_renderer.go
@@ -66,7 +66,6 @@ type sgmlRenderer struct {
 	sectionTitle              *textTemplate
 	sidebarBlock              *textTemplate
 	sourceBlock               *textTemplate
-	stringElement             *textTemplate
 	subscriptText             *textTemplate
 	superscriptText           *textTemplate
 	table                     *textTemplate
@@ -146,7 +145,6 @@ func (r *sgmlRenderer) prepareTemplates() error {
 		r.quoteParagraph, err = r.newTemplate("quote-paragraph", tmpls.QuoteParagraph, err)
 		r.sectionContent, err = r.newTemplate("section-content", tmpls.SectionContent, err)
 		r.sectionTitle, err = r.newTemplate("section-header", tmpls.SectionHeader, err)
-		r.stringElement, err = r.newTemplate("string-element", tmpls.StringElement, err)
 		r.sidebarBlock, err = r.newTemplate("sidebar-block", tmpls.SidebarBlock, err)
 		r.sourceBlock, err = r.newTemplate("source-block", tmpls.SourceBlock, err)
 		r.subscriptText, err = r.newTemplate("subscript", tmpls.SubscriptText, err)

diff --git a/pkg/renderer/sgml/string.go b/pkg/renderer/sgml/string.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/bytesparadise/libasciidoc/pkg/renderer"
 	"github.com/bytesparadise/libasciidoc/pkg/types"
-	"github.com/pkg/errors"
 )
 
 var quotes = map[types.QuotedStringKind]struct {
@@ -37,15 +36,9 @@ func (r *sgmlRenderer) renderQuotedString(ctx *renderer.Context, s *types.Quoted
 }
 
 func (r *sgmlRenderer) renderStringElement(ctx *renderer.Context, str *types.StringElement) (string, error) {
-	buf := &strings.Builder{}
-	err := r.stringElement.Execute(buf, str.Content)
-	if err != nil {
-		return "", errors.Wrap(err, "unable to render string")
-	}
-
 	// NB: For all SGML flavors we are aware of, the numeric entities from
 	// Unicode are supported.  We generally avoid named entities.
-	result := buf.String()
+	result := str.Content
 	if !ctx.UseUnicode() {
 		// convert to entities
 		result = asciiEntify(result)

diff --git a/pkg/renderer/sgml/templates.go b/pkg/renderer/sgml/templates.go
@@ -59,7 +59,6 @@ type Templates struct {
 	SectionHeader             string
 	SidebarBlock              string
 	SourceBlock               string
-	StringElement             string
 	SubscriptText             string
 	SuperscriptText           string
 	Table                     string