From 45a44a479d3879240122dba5bdba0c288245c56b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?=
 <bjorn.erik.pedersen@gmail.com>
Date: Tue, 23 Oct 2018 14:37:09 +0200
Subject: [PATCH] Resolve error handling/parser related TODOs

See #5324
---
 hugolib/hugo_sites_build_errors_test.go    |  2 +-
 hugolib/hugo_sites_build_test.go           |  8 ++--
 hugolib/page.go                            |  8 +++-
 hugolib/page_content.go                    | 21 ++++----
 hugolib/page_test.go                       | 12 ++---
 hugolib/site_render.go                     |  5 +-
 hugolib/site_test.go                       |  3 +-
 parser/pageparser/item.go                  |  5 +-
 parser/pageparser/pagelexer.go             | 56 +++++++++++-----------
 parser/pageparser/pageparser.go            |  2 +-
 parser/pageparser/pageparser_intro_test.go |  8 ++--
 tpl/template.go                            |  2 +-
 12 files changed, 66 insertions(+), 66 deletions(-)

diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go
index 1e53eb3c46b..8e913f061b2 100644
--- a/hugolib/hugo_sites_build_errors_test.go
+++ b/hugolib/hugo_sites_build_errors_test.go
@@ -191,7 +191,7 @@ func TestSiteBuildErrors(t *testing.T) {
 			},
 			assertBuildError: func(a testSiteBuildErrorAsserter, err error) {
 				assert.Error(err)
-				assert.Contains(err.Error(), `"content/mytoml.md": render of "page" failed: execute of template failed: panic in Execute`)
+				assert.Contains(err.Error(), `execute of template failed: panic in Execute`)
 			},
 		},
 	}
diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go
index 727cc6ed924..f1e317f5967 100644
--- a/hugolib/hugo_sites_build_test.go
+++ b/hugolib/hugo_sites_build_test.go
@@ -631,12 +631,10 @@ func assertShouldNotBuild(t *testing.T, sites *HugoSites) {
 	for _, p := range s.rawAllPages {
 		// No HTML when not processed
 		require.Equal(t, p.shouldBuild(), bytes.Contains(p.workContent, []byte("</")), p.BaseFileName()+": "+string(p.workContent))
-		// TODO(bep) 2errors
-		/*
-			require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild()))
 
-			require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName())
-		*/
+		require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild()))
+
+		require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName())
 
 	}
 }
diff --git a/hugolib/page.go b/hugolib/page.go
index df6f88b01ea..f7149ad7441 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -1685,9 +1685,13 @@ func (p *Page) shouldRenderTo(f output.Format) bool {
 	return found
 }
 
+// RawContent returns the un-rendered source content without
+// any leading front matter.
 func (p *Page) RawContent() string {
-	// TODO(bep) 2errors
-	return string(p.source.parsed.Input())
+	if p.source.posMainContent == -1 {
+		return ""
+	}
+	return string(p.source.parsed.Input()[p.source.posMainContent:])
 }
 
 func (p *Page) FullFilePath() string {
diff --git a/hugolib/page_content.go b/hugolib/page_content.go
index be015253bd9..dab46411e0e 100644
--- a/hugolib/page_content.go
+++ b/hugolib/page_content.go
@@ -46,11 +46,11 @@ type rawPageContent struct {
 	hasSummaryDivider bool
 
 	// The AST of the parsed page. Contains information about:
-	// shortcBackup3odes, front matter, summary indicators.
-	// TODO(bep) 2errors add this to a new rawPagecContent struct
-	// with frontMatterItem (pos) etc.
-	// * RawContent, RawContentWithoutFrontMatter
+	// shortcodes, front matter, summary indicators.
 	parsed pageparser.Result
+
+	// Returns the position in bytes after any front matter.
+	posMainContent int
 }
 
 // TODO(bep) lazy consolidate
@@ -58,6 +58,7 @@ func (p *Page) mapContent() error {
 	p.shortcodeState = newShortcodeHandler(p)
 	s := p.shortcodeState
 	p.renderable = true
+	p.source.posMainContent = -1
 
 	result := bp.GetBuffer()
 	defer bp.PutBuffer(result)
@@ -81,8 +82,8 @@ Loop:
 		case it.Type == pageparser.TypeIgnore:
 		case it.Type == pageparser.TypeHTMLComment:
 			// Ignore. This is only a leading Front matter comment.
-		case it.Type == pageparser.TypeHTMLDocument:
-			// This is HTML only. No shortcode, front matter etc.
+		case it.Type == pageparser.TypeHTMLStart:
+			// This is HTML without front matter. It can still have shortcodes.
 			p.renderable = false
 			result.Write(it.Val)
 		case it.IsFrontMatter():
@@ -99,12 +100,17 @@ Loop:
 				return err
 			}
 
+			next := iter.Peek()
+			if !next.IsDone() {
+				p.source.posMainContent = next.Pos
+			}
+
 			if !p.shouldBuild() {
 				// Nothing more to do.
 				return nil
 			}
 
-		case it.Type == pageparser.TypeLeadSummaryDivider, it.Type == pageparser.TypeSummaryDividerOrg:
+		case it.Type == pageparser.TypeLeadSummaryDivider:
 			result.Write(internalSummaryDivider)
 			p.source.hasSummaryDivider = true
 			// Need to determine if the page is truncated.
@@ -172,7 +178,6 @@ func (p *Page) parse(reader io.Reader) error {
 		parsed: parseResult,
 	}
 
-	// TODO(bep) 2errors
 	p.lang = p.File.Lang()
 
 	if p.s != nil && p.s.owner != nil {
diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index ced7e78d882..4499a97e077 100644
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -767,8 +767,7 @@ Simple Page With Some Date`
 }
 
 // Issue #2601
-// TODO(bep) 2errors
-func _TestPageRawContent(t *testing.T) {
+func TestPageRawContent(t *testing.T) {
 	t.Parallel()
 	cfg, fs := newTestCfg()
 
@@ -784,7 +783,7 @@ title: Raw
 	require.Len(t, s.RegularPages, 1)
 	p := s.RegularPages[0]
 
-	require.Contains(t, p.RawContent(), "**Raw**")
+	require.Equal(t, p.RawContent(), "**Raw**")
 
 }
 
@@ -1042,8 +1041,7 @@ func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) {
 	testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithAllCJKRunes)
 }
 
-// TODO(bep) 2errors
-func _TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) {
+func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) {
 	t.Parallel()
 	settings := map[string]interface{}{"hasCJKLanguage": true}
 
@@ -1056,8 +1054,7 @@ func _TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) {
 	testAllMarkdownEnginesForPages(t, assertFunc, settings, simplePageWithAllCJKRunes)
 }
 
-// TODO(bep) 2errors
-func _TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) {
+func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) {
 	t.Parallel()
 	settings := map[string]interface{}{"hasCJKLanguage": true}
 
@@ -1164,7 +1161,6 @@ func TestShouldRenderContent(t *testing.T) {
 		render bool
 	}{
 		{contentNoFrontmatter, true},
-		// TODO(bep) 2errors {invalidFrontmatterShortDelim, true},
 		{renderNoFrontmatter, false},
 		{contentWithCommentedFrontmatter, true},
 		{contentWithCommentedTextFrontmatter, true},
diff --git a/hugolib/site_render.go b/hugolib/site_render.go
index 6583acd06aa..a8cdca255e8 100644
--- a/hugolib/site_render.go
+++ b/hugolib/site_render.go
@@ -134,8 +134,7 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa
 
 			if shouldRender {
 				if err := pageOutput.renderResources(); err != nil {
-					// TODO(bep) 2errors
-					s.Log.ERROR.Printf("Failed to render resources for page %q: %s", page, err)
+					s.SendError(page.errorf(err, "failed to render page resources"))
 					continue
 				}
 			}
@@ -147,7 +146,7 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa
 			} else {
 				layouts, err = s.layouts(pageOutput)
 				if err != nil {
-					s.Log.ERROR.Printf("Failed to resolve layout output %q for page %q: %s", outFormat.Name, page, err)
+					s.Log.ERROR.Printf("Failed to resolve layout for output %q for page %q: %s", outFormat.Name, page, err)
 					continue
 				}
 			}
diff --git a/hugolib/site_test.go b/hugolib/site_test.go
index 2142025cc6b..0fd3a397a3c 100644
--- a/hugolib/site_test.go
+++ b/hugolib/site_test.go
@@ -451,8 +451,7 @@ func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) {
 
 }
 
-// TODO(bep) 2errors
-func _TestSkipRender(t *testing.T) {
+func TestSkipRender(t *testing.T) {
 	t.Parallel()
 	sources := [][2]string{
 		{filepath.FromSlash("sect/doc1.html"), "---\nmarkup: markdown\n---\n# title\nsome *content*"},
diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go
index 049db584afa..0567bd8b9c2 100644
--- a/parser/pageparser/item.go
+++ b/parser/pageparser/item.go
@@ -103,10 +103,9 @@ const (
 	tEOF
 
 	// page items
-	TypeHTMLDocument       // document starting with < as first non-whitespace
+	TypeHTMLStart          // document starting with < as first non-whitespace
 	TypeHTMLComment        // We ignore leading comments
-	TypeLeadSummaryDivider // <!--more-->
-	TypeSummaryDividerOrg  // # more
+	TypeLeadSummaryDivider // <!--more-->,  # more
 	TypeFrontMatterYAML
 	TypeFrontMatterTOML
 	TypeFrontMatterJSON
diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go
index e02475d420e..ddf109b3de0 100644
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -48,6 +48,8 @@ type pageLexer struct {
 	start      int // item start position
 	width      int // width of last element
 
+	// The summary divider to look for.
+	summaryDivider []byte
 	// Set when we have parsed any summary divider
 	summaryDividerChecked bool
 
@@ -69,7 +71,6 @@ func (l *pageLexer) Input() []byte {
 
 // note: the input position here is normally 0 (start), but
 // can be set if position of first shortcode is known
-// TODO(bep) 2errors byte
 func newPageLexer(input []byte, inputPosition int, stateStart stateFunc) *pageLexer {
 	lexer := &pageLexer{
 		input:      input,
@@ -117,7 +118,7 @@ var (
 	delimTOML         = []byte("+++")
 	delimYAML         = []byte("---")
 	delimOrg          = []byte("#+")
-	htmlCOmmentStart  = []byte("<!--")
+	htmlCommentStart  = []byte("<!--")
 	htmlCOmmentEnd    = []byte("-->")
 )
 
@@ -195,17 +196,18 @@ func (l *pageLexer) consumeCRLF() bool {
 
 func lexMainSection(l *pageLexer) stateFunc {
 	// Fast forward as far as possible.
-	var l1, l2, l3 int
-	if !l.summaryDividerChecked {
-		// TODO(bep) 2errors make the summary divider per type
-		l1 = l.index(summaryDivider)
-		l2 = l.index(summaryDividerOrg)
-		if l1 == -1 && l2 == -1 {
+	var l1, l2 int
+
+	if !l.summaryDividerChecked && l.summaryDivider != nil {
+		l1 = l.index(l.summaryDivider)
+		if l1 == -1 {
 			l.summaryDividerChecked = true
 		}
 	}
-	l3 = l.index(leftDelimSc)
-	skip := minPositiveIndex(l1, l2, l3)
+
+	l2 = l.index(leftDelimSc)
+	skip := minPositiveIndex(l1, l2)
+
 	if skip > 0 {
 		l.pos += skip
 	}
@@ -225,23 +227,14 @@ func lexMainSection(l *pageLexer) stateFunc {
 			return lexShortcodeLeftDelim
 		}
 
-		if !l.summaryDividerChecked {
-			if l.hasPrefix(summaryDivider) {
+		if !l.summaryDividerChecked && l.summaryDivider != nil {
+			if l.hasPrefix(l.summaryDivider) {
 				if l.pos > l.start {
 					l.emit(tText)
 				}
 				l.summaryDividerChecked = true
-				l.pos += len(summaryDivider)
-				//l.consumeCRLF()
+				l.pos += len(l.summaryDivider)
 				l.emit(TypeLeadSummaryDivider)
-			} else if l.hasPrefix(summaryDividerOrg) {
-				if l.pos > l.start {
-					l.emit(tText)
-				}
-				l.summaryDividerChecked = true
-				l.pos += len(summaryDividerOrg)
-				//l.consumeCRLF()
-				l.emit(TypeSummaryDividerOrg)
 			}
 		}
 
@@ -261,6 +254,8 @@ func (l *pageLexer) isShortCodeStart() bool {
 }
 
 func lexIntroSection(l *pageLexer) stateFunc {
+	l.summaryDivider = summaryDivider
+
 LOOP:
 	for {
 		r := l.next()
@@ -283,7 +278,7 @@ LOOP:
 			// No front matter.
 			if r == '<' {
 				l.backup()
-				if l.hasPrefix(htmlCOmmentStart) {
+				if l.hasPrefix(htmlCommentStart) {
 					right := l.index(htmlCOmmentEnd)
 					if right == -1 {
 						return l.errorf("starting HTML comment with no end")
@@ -291,10 +286,14 @@ LOOP:
 					l.pos += right + len(htmlCOmmentEnd)
 					l.emit(TypeHTMLComment)
 				} else {
-					// Not need to look further. Hugo treats this as plain HTML,
-					// no front matter, no shortcodes, no nothing.
-					l.pos = len(l.input)
-					l.emit(TypeHTMLDocument)
+					if l.pos > l.start {
+						l.emit(tText)
+					}
+					l.next()
+					// This is the start of a plain HTML document with no
+					// front matter. I still can contain shortcodes, so we
+					// have to keep looking.
+					l.emit(TypeHTMLStart)
 				}
 			}
 			break LOOP
@@ -365,10 +364,11 @@ func lexFrontMatterOrgMode(l *pageLexer) stateFunc {
 		#+DESCRIPTION: Just another golang parser for org content!
 	*/
 
+	l.summaryDivider = summaryDividerOrg
+
 	l.backup()
 
 	if !l.hasPrefix(delimOrg) {
-		// TODO(bep) consider error
 		return lexMainSection
 	}
 
diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go
index 6e75f195ade..75439712166 100644
--- a/parser/pageparser/pageparser.go
+++ b/parser/pageparser/pageparser.go
@@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) {
 }
 
 func parseMainSection(input []byte, from int) Result {
-	lexer := newPageLexer(input, from, lexMainSection) // TODO(bep) 2errors
+	lexer := newPageLexer(input, from, lexMainSection)
 	lexer.run()
 	return lexer
 }
diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go
index 32de6dc4446..ba4a2c84b76 100644
--- a/parser/pageparser/pageparser_intro_test.go
+++ b/parser/pageparser/pageparser_intro_test.go
@@ -38,7 +38,7 @@ var (
 	tstFrontMatterJSON     = nti(TypeFrontMatterJSON, tstJSON+"\r\n")
 	tstSomeText            = nti(tText, "\nSome text.\n")
 	tstSummaryDivider      = nti(TypeLeadSummaryDivider, "<!--more-->")
-	tstSummaryDividerOrg   = nti(TypeSummaryDividerOrg, "# more")
+	tstHtmlStart           = nti(TypeHTMLStart, "<")
 
 	tstORG = `
 #+TITLE: T1
@@ -54,8 +54,8 @@ var crLfReplacer = strings.NewReplacer("\r", "#", "\n", "$")
 var frontMatterTests = []lexerTest{
 	{"empty", "", []Item{tstEOF}},
 	{"Byte order mark", "\ufeff\nSome text.\n", []Item{nti(TypeIgnore, "\ufeff"), tstSomeText, tstEOF}},
-	{"HTML Document", `  <html>  `, []Item{nti(TypeHTMLDocument, "  <html>  "), tstEOF}},
-	{"HTML Document 2", `<html><h1>Hugo Rocks</h1></html>`, []Item{nti(TypeHTMLDocument, "<html><h1>Hugo Rocks</h1></html>"), tstEOF}},
+	{"HTML Document", `  <html>  `, []Item{nti(tText, "  "), tstHtmlStart, nti(tText, "html>  "), tstEOF}},
+	{"HTML Document with shortcode", `<html>{{< sc1 >}}</html>`, []Item{tstHtmlStart, nti(tText, "html>"), tstLeftNoMD, tstSC1, tstRightNoMD, nti(tText, "</html>"), tstEOF}},
 	{"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}},
 	{"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}},
 	{"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}},
@@ -65,7 +65,7 @@ var frontMatterTests = []lexerTest{
 	{"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}},
 	{"JSON front matter", tstJSON + "\r\n\nSome text.\n", []Item{tstFrontMatterJSON, tstSomeText, tstEOF}},
 	{"ORG front matter", tstORG + "\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, tstEOF}},
-	{"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, tstSummaryDividerOrg, tstSomeText, tstEOF}},
+	{"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more"), tstSomeText, tstEOF}},
 	{"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, tstSomeText, tstEOF}},
 }
 
diff --git a/tpl/template.go b/tpl/template.go
index 12a4607fbf7..9687054934a 100644
--- a/tpl/template.go
+++ b/tpl/template.go
@@ -179,7 +179,7 @@ func (t *TemplateAdapter) addFileContext(name string, inerr error) error {
 		}
 		return false
 	}
-	// TODO(bep) 2errors text vs HTML
+
 	fe, ok := herrors.WithFileContext(ferr, realFilename, f, lineMatcher)
 	if ok || !hasMaster {
 		return fe