diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index d76e00d2e4d..e6a0301c941 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -50,7 +50,7 @@ func TestRootMappingFsDirnames(t *testing.T) { fif, err := rfs.Stat(filepath.Join("cf2", testfile)) assert.NoError(err) assert.Equal("myfile.txt", fif.Name()) - assert.Equal("f2t/myfile.txt", fif.(RealFilenameInfo).RealFilename()) + assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fif.(RealFilenameInfo).RealFilename()) root, err := rfs.Open(filepathSeparator) assert.NoError(err) diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index 1e53eb3c46b..8e913f061b2 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -191,7 +191,7 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { assert.Error(err) - assert.Contains(err.Error(), `"content/mytoml.md": render of "page" failed: execute of template failed: panic in Execute`) + assert.Contains(err.Error(), `execute of template failed: panic in Execute`) }, }, } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 727cc6ed924..f1e317f5967 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -631,12 +631,10 @@ func assertShouldNotBuild(t *testing.T, sites *HugoSites) { for _, p := range s.rawAllPages { // No HTML when not processed require.Equal(t, p.shouldBuild(), bytes.Contains(p.workContent, []byte("")), p.BaseFileName()+": "+string(p.workContent)) - // TODO(bep) 2errors - /* - require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild())) - require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName()) - */ + require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild())) + + require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName()) } } diff --git a/hugolib/page.go b/hugolib/page.go index df6f88b01ea..f7149ad7441 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -1685,9 +1685,13 @@ func (p *Page) shouldRenderTo(f output.Format) bool { return found } +// RawContent returns the un-rendered source content without +// any leading front matter. func (p *Page) RawContent() string { - // TODO(bep) 2errors - return string(p.source.parsed.Input()) + if p.source.posMainContent == -1 { + return "" + } + return string(p.source.parsed.Input()[p.source.posMainContent:]) } func (p *Page) FullFilePath() string { diff --git a/hugolib/page_content.go b/hugolib/page_content.go index be015253bd9..dab46411e0e 100644 --- a/hugolib/page_content.go +++ b/hugolib/page_content.go @@ -46,11 +46,11 @@ type rawPageContent struct { hasSummaryDivider bool // The AST of the parsed page. Contains information about: - // shortcBackup3odes, front matter, summary indicators. - // TODO(bep) 2errors add this to a new rawPagecContent struct - // with frontMatterItem (pos) etc. - // * RawContent, RawContentWithoutFrontMatter + // shortcodes, front matter, summary indicators. parsed pageparser.Result + + // Returns the position in bytes after any front matter. + posMainContent int } // TODO(bep) lazy consolidate @@ -58,6 +58,7 @@ func (p *Page) mapContent() error { p.shortcodeState = newShortcodeHandler(p) s := p.shortcodeState p.renderable = true + p.source.posMainContent = -1 result := bp.GetBuffer() defer bp.PutBuffer(result) @@ -81,8 +82,8 @@ Loop: case it.Type == pageparser.TypeIgnore: case it.Type == pageparser.TypeHTMLComment: // Ignore. This is only a leading Front matter comment. - case it.Type == pageparser.TypeHTMLDocument: - // This is HTML only. No shortcode, front matter etc. + case it.Type == pageparser.TypeHTMLStart: + // This is HTML without front matter. It can still have shortcodes. p.renderable = false result.Write(it.Val) case it.IsFrontMatter(): @@ -99,12 +100,17 @@ Loop: return err } + next := iter.Peek() + if !next.IsDone() { + p.source.posMainContent = next.Pos + } + if !p.shouldBuild() { // Nothing more to do. return nil } - case it.Type == pageparser.TypeLeadSummaryDivider, it.Type == pageparser.TypeSummaryDividerOrg: + case it.Type == pageparser.TypeLeadSummaryDivider: result.Write(internalSummaryDivider) p.source.hasSummaryDivider = true // Need to determine if the page is truncated. @@ -172,7 +178,6 @@ func (p *Page) parse(reader io.Reader) error { parsed: parseResult, } - // TODO(bep) 2errors p.lang = p.File.Lang() if p.s != nil && p.s.owner != nil { diff --git a/hugolib/page_test.go b/hugolib/page_test.go index ced7e78d882..4499a97e077 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -767,8 +767,7 @@ Simple Page With Some Date` } // Issue #2601 -// TODO(bep) 2errors -func _TestPageRawContent(t *testing.T) { +func TestPageRawContent(t *testing.T) { t.Parallel() cfg, fs := newTestCfg() @@ -784,7 +783,7 @@ title: Raw require.Len(t, s.RegularPages, 1) p := s.RegularPages[0] - require.Contains(t, p.RawContent(), "**Raw**") + require.Equal(t, p.RawContent(), "**Raw**") } @@ -1042,8 +1041,7 @@ func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithAllCJKRunes) } -// TODO(bep) 2errors -func _TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { +func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { t.Parallel() settings := map[string]interface{}{"hasCJKLanguage": true} @@ -1056,8 +1054,7 @@ func _TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { testAllMarkdownEnginesForPages(t, assertFunc, settings, simplePageWithAllCJKRunes) } -// TODO(bep) 2errors -func _TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { +func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { t.Parallel() settings := map[string]interface{}{"hasCJKLanguage": true} @@ -1164,7 +1161,6 @@ func TestShouldRenderContent(t *testing.T) { render bool }{ {contentNoFrontmatter, true}, - // TODO(bep) 2errors {invalidFrontmatterShortDelim, true}, {renderNoFrontmatter, false}, {contentWithCommentedFrontmatter, true}, {contentWithCommentedTextFrontmatter, true}, diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 6583acd06aa..a8cdca255e8 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -134,8 +134,7 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa if shouldRender { if err := pageOutput.renderResources(); err != nil { - // TODO(bep) 2errors - s.Log.ERROR.Printf("Failed to render resources for page %q: %s", page, err) + s.SendError(page.errorf(err, "failed to render page resources")) continue } } @@ -147,7 +146,7 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa } else { layouts, err = s.layouts(pageOutput) if err != nil { - s.Log.ERROR.Printf("Failed to resolve layout output %q for page %q: %s", outFormat.Name, page, err) + s.Log.ERROR.Printf("Failed to resolve layout for output %q for page %q: %s", outFormat.Name, page, err) continue } } diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 2142025cc6b..0fd3a397a3c 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -451,8 +451,7 @@ func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) { } -// TODO(bep) 2errors -func _TestSkipRender(t *testing.T) { +func TestSkipRender(t *testing.T) { t.Parallel() sources := [][2]string{ {filepath.FromSlash("sect/doc1.html"), "---\nmarkup: markdown\n---\n# title\nsome *content*"}, diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go index 049db584afa..0567bd8b9c2 100644 --- a/parser/pageparser/item.go +++ b/parser/pageparser/item.go @@ -103,10 +103,9 @@ const ( tEOF // page items - TypeHTMLDocument // document starting with < as first non-whitespace + TypeHTMLStart // document starting with < as first non-whitespace TypeHTMLComment // We ignore leading comments - TypeLeadSummaryDivider // - TypeSummaryDividerOrg // # more + TypeLeadSummaryDivider // , # more TypeFrontMatterYAML TypeFrontMatterTOML TypeFrontMatterJSON diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index e02475d420e..ddf109b3de0 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -48,6 +48,8 @@ type pageLexer struct { start int // item start position width int // width of last element + // The summary divider to look for. + summaryDivider []byte // Set when we have parsed any summary divider summaryDividerChecked bool @@ -69,7 +71,6 @@ func (l *pageLexer) Input() []byte { // note: the input position here is normally 0 (start), but // can be set if position of first shortcode is known -// TODO(bep) 2errors byte func newPageLexer(input []byte, inputPosition int, stateStart stateFunc) *pageLexer { lexer := &pageLexer{ input: input, @@ -117,7 +118,7 @@ var ( delimTOML = []byte("+++") delimYAML = []byte("---") delimOrg = []byte("#+") - htmlCOmmentStart = []byte("") ) @@ -195,17 +196,18 @@ func (l *pageLexer) consumeCRLF() bool { func lexMainSection(l *pageLexer) stateFunc { // Fast forward as far as possible. - var l1, l2, l3 int - if !l.summaryDividerChecked { - // TODO(bep) 2errors make the summary divider per type - l1 = l.index(summaryDivider) - l2 = l.index(summaryDividerOrg) - if l1 == -1 && l2 == -1 { + var l1, l2 int + + if !l.summaryDividerChecked && l.summaryDivider != nil { + l1 = l.index(l.summaryDivider) + if l1 == -1 { l.summaryDividerChecked = true } } - l3 = l.index(leftDelimSc) - skip := minPositiveIndex(l1, l2, l3) + + l2 = l.index(leftDelimSc) + skip := minPositiveIndex(l1, l2) + if skip > 0 { l.pos += skip } @@ -225,23 +227,14 @@ func lexMainSection(l *pageLexer) stateFunc { return lexShortcodeLeftDelim } - if !l.summaryDividerChecked { - if l.hasPrefix(summaryDivider) { + if !l.summaryDividerChecked && l.summaryDivider != nil { + if l.hasPrefix(l.summaryDivider) { if l.pos > l.start { l.emit(tText) } l.summaryDividerChecked = true - l.pos += len(summaryDivider) - //l.consumeCRLF() + l.pos += len(l.summaryDivider) l.emit(TypeLeadSummaryDivider) - } else if l.hasPrefix(summaryDividerOrg) { - if l.pos > l.start { - l.emit(tText) - } - l.summaryDividerChecked = true - l.pos += len(summaryDividerOrg) - //l.consumeCRLF() - l.emit(TypeSummaryDividerOrg) } } @@ -261,6 +254,8 @@ func (l *pageLexer) isShortCodeStart() bool { } func lexIntroSection(l *pageLexer) stateFunc { + l.summaryDivider = summaryDivider + LOOP: for { r := l.next() @@ -283,7 +278,7 @@ LOOP: // No front matter. if r == '<' { l.backup() - if l.hasPrefix(htmlCOmmentStart) { + if l.hasPrefix(htmlCommentStart) { right := l.index(htmlCOmmentEnd) if right == -1 { return l.errorf("starting HTML comment with no end") @@ -291,10 +286,14 @@ LOOP: l.pos += right + len(htmlCOmmentEnd) l.emit(TypeHTMLComment) } else { - // Not need to look further. Hugo treats this as plain HTML, - // no front matter, no shortcodes, no nothing. - l.pos = len(l.input) - l.emit(TypeHTMLDocument) + if l.pos > l.start { + l.emit(tText) + } + l.next() + // This is the start of a plain HTML document with no + // front matter. I still can contain shortcodes, so we + // have to keep looking. + l.emit(TypeHTMLStart) } } break LOOP @@ -365,10 +364,11 @@ func lexFrontMatterOrgMode(l *pageLexer) stateFunc { #+DESCRIPTION: Just another golang parser for org content! */ + l.summaryDivider = summaryDividerOrg + l.backup() if !l.hasPrefix(delimOrg) { - // TODO(bep) consider error return lexMainSection } diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go index 6e75f195ade..75439712166 100644 --- a/parser/pageparser/pageparser.go +++ b/parser/pageparser/pageparser.go @@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) { } func parseMainSection(input []byte, from int) Result { - lexer := newPageLexer(input, from, lexMainSection) // TODO(bep) 2errors + lexer := newPageLexer(input, from, lexMainSection) lexer.run() return lexer } diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go index 32de6dc4446..ba4a2c84b76 100644 --- a/parser/pageparser/pageparser_intro_test.go +++ b/parser/pageparser/pageparser_intro_test.go @@ -38,7 +38,7 @@ var ( tstFrontMatterJSON = nti(TypeFrontMatterJSON, tstJSON+"\r\n") tstSomeText = nti(tText, "\nSome text.\n") tstSummaryDivider = nti(TypeLeadSummaryDivider, "") - tstSummaryDividerOrg = nti(TypeSummaryDividerOrg, "# more") + tstHtmlStart = nti(TypeHTMLStart, "<") tstORG = ` #+TITLE: T1 @@ -54,8 +54,8 @@ var crLfReplacer = strings.NewReplacer("\r", "#", "\n", "$") var frontMatterTests = []lexerTest{ {"empty", "", []Item{tstEOF}}, {"Byte order mark", "\ufeff\nSome text.\n", []Item{nti(TypeIgnore, "\ufeff"), tstSomeText, tstEOF}}, - {"HTML Document", ` `, []Item{nti(TypeHTMLDocument, " "), tstEOF}}, - {"HTML Document 2", `