Fix handling of HTML files without front matter

This means that any HTML file inside /content will be treated as a regular file.

If you want it processes with shortcodes and a layout, add front matter.

The defintion of an HTML file here is:

* File with extension .htm or .html
* With first non-whitespace character "<" that isn't a HTML comment.

This is in line with the documentation.

Fixes gohugoio#7030
Fixes gohugoio#7028
See gohugoio#6789

deps/deps.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"go.uber.org/atomic"
 
 	"github.com/gohugoio/hugo/cache/filecache"
 	"github.com/gohugoio/hugo/common/loggers"
@@ -377,11 +376,8 @@ type DepsCfg struct {
 
 // BuildFlags are flags that may be turned on during a build.
 type BuildFlags struct {
-	HasLateTemplate atomic.Bool
 }
 
 func NewBuildFlags() BuildFlags {
-	return BuildFlags{
-		//HasLateTemplate: atomic.NewBool(false),
-	}
+	return BuildFlags{}
 }

deps/deps_test.go

@@ -15,14 +15,8 @@ package deps
 
 import (
 	"testing"
-
-	qt "github.com/frankban/quicktest"
 )
 
 func TestBuildFlags(t *testing.T) {
-	c := qt.New(t)
-	var bf BuildFlags
-	c.Assert(bf.HasLateTemplate.Load(), qt.Equals, false)
-	bf.HasLateTemplate.Store(true)
-	c.Assert(bf.HasLateTemplate.Load(), qt.Equals, true)
+
 }

hugofs/files/classifier.go

@@ -14,10 +14,16 @@
 package files
 
 import (
+	"bufio"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+	"unicode"
+
+	"github.com/spf13/afero"
 )
 
 var (
@@ -32,19 +38,32 @@ var (
 		"pandoc", "pdc"}
 
 	contentFileExtensionsSet map[string]bool
+
+	htmlFileExtensions = []string{
+		"html", "htm"}
+
+	htmlFileExtensionsSet map[string]bool
 )
 
 func init() {
 	contentFileExtensionsSet = make(map[string]bool)
 	for _, ext := range contentFileExtensions {
 		contentFileExtensionsSet[ext] = true
 	}
+	htmlFileExtensionsSet = make(map[string]bool)
+	for _, ext := range htmlFileExtensions {
+		htmlFileExtensionsSet[ext] = true
+	}
 }
 
 func IsContentFile(filename string) bool {
 	return contentFileExtensionsSet[strings.TrimPrefix(filepath.Ext(filename), ".")]
 }
 
+func IsHTMLFile(filename string) bool {
+	return htmlFileExtensionsSet[strings.TrimPrefix(filepath.Ext(filename), ".")]
+}
+
 func IsContentExt(ext string) bool {
 	return contentFileExtensionsSet[ext]
 }
@@ -62,10 +81,33 @@ func (c ContentClass) IsBundle() bool {
 	return c == ContentClassLeaf || c == ContentClassBranch
 }
 
-func ClassifyContentFile(filename string) ContentClass {
+func ClassifyContentFile(filename string, open func() (afero.File, error)) ContentClass {
 	if !IsContentFile(filename) {
 		return ContentClassFile
 	}
+
+	if IsHTMLFile(filename) {
+		// We need to look inside the file. If the first non-whitespace
+		// character is a "<", then we treat it as a regular file.
+		// Eearlier we created pages for these files, but that had all sorts
+		// of troubles, and isn't what it says in the documentation.
+		// See https://github.com/gohugoio/hugo/issues/7030
+		if open == nil {
+			panic(fmt.Sprintf("no file opener provided for %q", filename))
+		}
+
+		f, err := open()
+		if err != nil {
+			return ContentClassFile
+		}
+		ishtml := isHTMLContent(f)
+		f.Close()
+		if ishtml {
+			return ContentClassFile
+		}
+
+	}
+
 	if strings.HasPrefix(filename, "_index.") {
 		return ContentClassBranch
 	}
@@ -77,6 +119,40 @@ func ClassifyContentFile(filename string) ContentClass {
 	return ContentClassContent
 }
 
+var htmlComment = []rune{'<', '!', '-', '-'}
+
+func isHTMLContent(r io.Reader) bool {
+	br := bufio.NewReader(r)
+	i := 0
+	for {
+		c, _, err := br.ReadRune()
+		if err != nil {
+			break
+		}
+
+		if i > 0 {
+			if i >= len(htmlComment) {
+				return false
+			}
+
+			if c != htmlComment[i] {
+				return true
+			}
+
+			i++
+			continue
+		}
+
+		if !unicode.IsSpace(c) {
+			if i == 0 && c != '<' {
+				return false
+			}
+			i++
+		}
+	}
+	return true
+}
+
 const (
 	ComponentFolderArchetypes = "archetypes"
 	ComponentFolderStatic     = "static"

hugofs/files/classifier_test.go

@@ -15,6 +15,7 @@ package files
 
 import (
 	"path/filepath"
+	"strings"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
@@ -30,6 +31,17 @@ func TestIsContentFile(t *testing.T) {
 	c.Assert(IsContentExt("json"), qt.Equals, false)
 }
 
+func TestIsHTMLContent(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(isHTMLContent(strings.NewReader("   <html>")), qt.Equals, true)
+	c.Assert(isHTMLContent(strings.NewReader("   <!--\n---")), qt.Equals, false)
+	c.Assert(isHTMLContent(strings.NewReader("   <!--")), qt.Equals, true)
+	c.Assert(isHTMLContent(strings.NewReader("   ---<")), qt.Equals, false)
+	c.Assert(isHTMLContent(strings.NewReader(" foo  <")), qt.Equals, false)
+
+}
+
 func TestComponentFolders(t *testing.T) {
 	c := qt.New(t)
 

hugofs/filter_fs.go

@@ -66,7 +66,7 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) {
 				metaKeyOrdinal:                    langs[lang],
 				metaKeyTranslationBaseName:        translationBaseName,
 				metaKeyTranslationBaseNameWithExt: translationBaseNameWithExt,
-				metaKeyClassifier:                 files.ClassifyContentFile(fi.Name()),
+				metaKeyClassifier:                 files.ClassifyContentFile(fi.Name(), meta.GetOpener()),
 			})
 
 			fis[i] = fim

hugolib/content_map_test.go

@@ -54,7 +54,7 @@ func BenchmarkContentMap(b *testing.B) {
 				// real flow, so simulate this here.
 				meta["lang"] = lang
 				meta["path"] = meta.Filename()
-				meta["classifier"] = files.ClassifyContentFile(fi.Name())
+				meta["classifier"] = files.ClassifyContentFile(fi.Name(), meta.GetOpener())
 
 			})
 	}
@@ -115,7 +115,7 @@ func TestContentMap(t *testing.T) {
 				// real flow, so simulate this here.
 				meta["lang"] = lang
 				meta["path"] = meta.Filename()
-				meta["classifier"] = files.ClassifyContentFile(fi.Name())
+				meta["classifier"] = files.ClassifyContentFile(fi.Name(), meta.GetOpener())
 				meta["translationBaseName"] = helpers.Filename(fi.Name())
 
 			})

hugolib/content_render_hooks_test.go

@@ -135,7 +135,7 @@ title: No Template
 	}
 	counters := &testCounters{}
 	b.Build(BuildCfg{testCounters: counters})
-	b.Assert(int(counters.contentRenderCounter), qt.Equals, 50)
+	b.Assert(int(counters.contentRenderCounter), qt.Equals, 43)
 
 	b.AssertFileContent("public/blog/p1/index.html", `
 <p>Cool Page|https://www.google.com|Title: Google's Homepage|Text: First Link|END</p>

hugolib/page.go

@@ -449,12 +449,6 @@ func (p *pageState) initOutputFormat(isRenderingSite bool, idx int) error {
 		return err
 	}
 
-	if !p.renderable {
-		if _, err := p.Content(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 
 }
@@ -679,8 +673,6 @@ func (p *pageState) mapContent(bucket *pagesMapBucket, meta *pageMeta) error {
 
 	s := p.shortcodeState
 
-	p.renderable = true
-
 	rn := &pageContentMap{
 		items: make([]interface{}, 0, 20),
 	}
@@ -703,12 +695,6 @@ Loop:
 
 		switch {
 		case it.Type == pageparser.TypeIgnore:
-		case it.Type == pageparser.TypeHTMLStart:
-			// This is HTML without front matter. It can still have shortcodes.
-			p.selfLayout = "__" + p.File().Filename()
-			p.renderable = false
-			p.s.BuildFlags.HasLateTemplate.CAS(false, true)
-			rn.AddBytes(it)
 		case it.IsFrontMatter():
 			f := pageparser.FormatFromFrontMatterType(it.Type)
 			m, err := metadecoders.Default.UnmarshalToMap(it.Val, f)

hugolib/page__content.go

@@ -28,7 +28,6 @@ var (
 
 // The content related items on a Page.
 type pageContent struct {
-	renderable bool
 	selfLayout string
 	truncated  bool
 
@@ -52,7 +51,7 @@ func (p pageContent) contentToRender(renderedShortcodes map[string]string) []byt
 		case pageContentReplacement:
 			c = append(c, v.val...)
 		case *shortcode:
-			if !p.renderable || !v.insertPlaceholder() {
+			if !v.insertPlaceholder() {
 				// Insert the rendered shortcode.
 				renderedShortcode, found := renderedShortcodes[v.placeholder]
 				if !found {