tpl/tplimpl: Rework template management to get rid of concurrency issues

This more or less completes the simplification of the template handling code in Hugo started in v0.62. The main motivation was to fix a long lasting issue about a crash in HTML content files without front matter. But this commit also comes with a big functional improvement. As we now have moved the base template evaluation to the build stage we now use the same lookup rules for `baseof` as for `list` etc. type of templates. This means that in this simple example you can have a `baseof` template for the `blog` section without having to duplicate the others: ``` layouts ├── _default │ ├── baseof.html │ ├── list.html │ └── single.html └── blog └── baseof.html ``` Also, when simplifying code, you often get rid of some double work, as shown in the "site building" benchmarks below. These benchmarks looks suspiciously good, but I have repeated the below with ca. the same result. Compared to master: ``` name old time/op new time/op delta SiteNew/Bundle_with_image-16 13.1ms ± 1% 10.5ms ± 1% -19.34% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 13.0ms ± 0% 10.7ms ± 1% -18.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 46.4ms ± 2% 43.1ms ± 1% -7.15% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 52.2ms ± 2% 47.8ms ± 1% -8.30% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 77.9ms ± 1% 70.9ms ± 1% -9.01% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 43.0ms ± 0% 37.2ms ± 1% -13.54% (p=0.029 n=4+4) SiteNew/Page_collections-16 58.2ms ± 1% 52.4ms ± 1% -9.95% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image-16 3.81MB ± 0% 2.22MB ± 0% -41.70% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 3.60MB ± 0% 2.01MB ± 0% -44.20% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 19.3MB ± 1% 14.1MB ± 0% -26.91% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 70.7MB ± 0% 69.0MB ± 0% -2.40% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 37.1MB ± 0% 31.2MB ± 0% -15.94% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 17.6MB ± 0% 10.6MB ± 0% -39.92% (p=0.029 n=4+4) SiteNew/Page_collections-16 25.9MB ± 0% 21.2MB ± 0% -17.99% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image-16 52.3k ± 0% 26.1k ± 0% -50.18% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 52.3k ± 0% 26.1k ± 0% -50.16% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 336k ± 1% 269k ± 0% -19.90% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 422k ± 0% 395k ± 0% -6.43% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 401k ± 0% 313k ± 0% -21.79% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 247k ± 0% 143k ± 0% -42.17% (p=0.029 n=4+4) SiteNew/Page_collections-16 282k ± 0% 207k ± 0% -26.55% (p=0.029 n=4+4) ``` Fixes gohugoio#6716 Fixes gohugoio#6760 Fixes gohugoio#6768 Fixes gohugoio#6778
jakejarvis · Mar 5, 2021 · ec35fd2 · ec35fd2
1 parent 9269357
commit ec35fd2
Show file tree

Hide file tree

Showing 46 changed files with 1,258 additions and 1,372 deletions.
diff --git a/commands/commandeer.go b/commands/commandeer.go
@@ -133,7 +133,7 @@ func (c *commandeer) getErrorWithContext() interface{} {
 
 	if c.h.verbose {
 		var b bytes.Buffer
-		herrors.FprintStackTrace(&b, c.buildErr)
+		herrors.FprintStackTraceFromErr(&b, c.buildErr)
 		m["StackTrace"] = b.String()
 	}
 

diff --git a/commands/hugo.go b/commands/hugo.go
@@ -716,7 +716,7 @@ func (c *commandeer) handleBuildErr(err error, msg string) {
 	c.logger.ERROR.Print(msg + ":\n\n")
 	c.logger.ERROR.Println(helpers.FirstUpper(err.Error()))
 	if !c.h.quiet && c.h.verbose {
-		herrors.PrintStackTrace(err)
+		herrors.PrintStackTraceFromErr(err)
 	}
 }
 

diff --git a/commands/server.go b/commands/server.go
@@ -416,7 +416,7 @@ func (c *commandeer) serve(s *serverCmd) error {
 		roots = []string{""}
 	}
 
-	templ, err := c.hugo().TextTmpl.Parse("__default_server_error", buildErrorTemplate)
+	templ, err := c.hugo().TextTmpl().Parse("__default_server_error", buildErrorTemplate)
 	if err != nil {
 		return err
 	}
@@ -428,7 +428,7 @@ func (c *commandeer) serve(s *serverCmd) error {
 		s:        s,
 		errorTemplate: func(ctx interface{}) (io.Reader, error) {
 			b := &bytes.Buffer{}
-			err := c.hugo().Tmpl.Execute(templ, b, ctx)
+			err := c.hugo().Tmpl().Execute(templ, b, ctx)
 			return b, err
 		},
 	}

diff --git a/common/herrors/errors.go b/common/herrors/errors.go
@@ -15,11 +15,14 @@
 package herrors
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
 	"os"
+	"runtime"
 	"runtime/debug"
+	"strconv"
 
 	_errors "github.com/pkg/errors"
 )
@@ -33,20 +36,27 @@ type stackTracer interface {
 	StackTrace() _errors.StackTrace
 }
 
-// PrintStackTrace prints the error's stack trace to stdoud.
-func PrintStackTrace(err error) {
-	FprintStackTrace(os.Stdout, err)
+// PrintStackTraceFromErr prints the error's stack trace to stdoud.
+func PrintStackTraceFromErr(err error) {
+	FprintStackTraceFromErr(os.Stdout, err)
 }
 
-// FprintStackTrace prints the error's stack trace to w.
-func FprintStackTrace(w io.Writer, err error) {
+// FprintStackTraceFromErr prints the error's stack trace to w.
+func FprintStackTraceFromErr(w io.Writer, err error) {
 	if err, ok := err.(stackTracer); ok {
 		for _, f := range err.StackTrace() {
 			fmt.Fprintf(w, "%+s:%d\n", f, f)
 		}
 	}
 }
 
+// PrintStackTrace prints the current stacktrace to w.
+func PrintStackTrace(w io.Writer) {
+	buf := make([]byte, 1<<16)
+	runtime.Stack(buf, true)
+	fmt.Fprintf(w, "%s", buf)
+}
+
 // Recover is a helper function that can be used to capture panics.
 // Put this at the top of a method/function that crashes in a template:
 //     defer herrors.Recover()
@@ -56,7 +66,16 @@ func Recover(args ...interface{}) {
 		args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n")
 		fmt.Println(args...)
 	}
+}
 
+// Get the current goroutine id. Used only for debugging.
+func GetGID() uint64 {
+	b := make([]byte, 64)
+	b = b[:runtime.Stack(b, false)]
+	b = bytes.TrimPrefix(b, []byte("goroutine "))
+	b = b[:bytes.IndexByte(b, ' ')]
+	n, _ := strconv.ParseUint(string(b), 10, 64)
+	return n
 }
 
 // ErrFeatureNotAvailable denotes that a feature is unavailable.

diff --git a/common/types/types.go b/common/types/types.go
@@ -21,6 +21,12 @@ import (
 	"github.com/spf13/cast"
 )
 
+// RLocker represents the read locks in sync.RWMutex.
+type RLocker interface {
+	RLock()
+	RUnlock()
+}
+
 // KeyValueStr is a string tuple.
 type KeyValueStr struct {
 	Key   string

diff --git a/create/content_template_handler.go b/create/content_template_handler.go
@@ -129,9 +129,9 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archety
 	archetypeTemplate = []byte(archetypeShortcodeReplacementsPre.Replace(string(archetypeTemplate)))
 
 	// Reuse the Hugo template setup to get the template funcs properly set up.
-	templateHandler := s.Deps.Tmpl.(tpl.TemplateManager)
-	templateName := "_text/" + helpers.Filename(archetypeFilename)
-	if err := templateHandler.AddTemplate(templateName, string(archetypeTemplate)); err != nil {
+	templateHandler := s.Deps.Tmpl().(tpl.TemplateManager)
+	templateName := helpers.Filename(archetypeFilename)
+	if err := templateHandler.AddTemplate("_text/"+templateName, string(archetypeTemplate)); err != nil {
 		return nil, errors.Wrapf(err, "Failed to parse archetype file %q:", archetypeFilename)
 	}
 

diff --git a/deps/deps.go b/deps/deps.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"go.uber.org/atomic"
 
 	"github.com/gohugoio/hugo/cache/filecache"
 	"github.com/gohugoio/hugo/common/loggers"
@@ -38,10 +39,10 @@ type Deps struct {
 	DistinctWarningLog *helpers.DistinctLogger
 
 	// The templates to use. This will usually implement the full tpl.TemplateManager.
-	Tmpl tpl.TemplateHandler `json:"-"`
+	tmpl tpl.TemplateHandler
 
 	// We use this to parse and execute ad-hoc text templates.
-	TextTmpl tpl.TemplateParseFinder `json:"-"`
+	textTmpl tpl.TemplateParseFinder
 
 	// The file systems to use.
 	Fs *hugofs.Fs `json:"-"`
@@ -92,6 +93,9 @@ type Deps struct {
 	// BuildStartListeners will be notified before a build starts.
 	BuildStartListeners *Listeners
 
+	// Atomic flags set during a build.
+	BuildFlags BuildFlags
+
 	*globalErrHandler
 }
 
@@ -153,9 +157,20 @@ type ResourceProvider interface {
 	Clone(deps *Deps) error
 }
 
-// TemplateHandler returns the used tpl.TemplateFinder as tpl.TemplateHandler.
-func (d *Deps) TemplateHandler() tpl.TemplateManager {
-	return d.Tmpl.(tpl.TemplateManager)
+func (d *Deps) Tmpl() tpl.TemplateHandler {
+	return d.tmpl
+}
+
+func (d *Deps) TextTmpl() tpl.TemplateParseFinder {
+	return d.textTmpl
+}
+
+func (d *Deps) SetTmpl(tmpl tpl.TemplateHandler) {
+	d.tmpl = tmpl
+}
+
+func (d *Deps) SetTextTmpl(tmpl tpl.TemplateParseFinder) {
+	d.textTmpl = tmpl
 }
 
 // LoadResources loads translations and templates.
@@ -315,6 +330,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er
 	}
 
 	d.BuildStartListeners = &Listeners{}
+	d.BuildFlags = BuildFlags{}
 
 	return &d, nil
 
@@ -358,3 +374,14 @@ type DepsCfg struct {
 	// Whether we are in running (server) mode
 	Running bool
 }
+
+// BuildFlags are flags that may be turned on during a build.
+type BuildFlags struct {
+	HasLateTemplate atomic.Bool
+}
+
+func NewBuildFlags() BuildFlags {
+	return BuildFlags{
+		//HasLateTemplate: atomic.NewBool(false),
+	}
+}
diff --git a/deps/deps_test.go b/deps/deps_test.go
@@ -0,0 +1,28 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deps
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestBuildFlags(t *testing.T) {
+	c := qt.New(t)
+	var bf BuildFlags
+	c.Assert(bf.HasLateTemplate.Load(), qt.Equals, false)
+	bf.HasLateTemplate.Store(true)
+	c.Assert(bf.HasLateTemplate.Load(), qt.Equals, true)
+}
diff --git a/go.mod b/go.mod
@@ -55,6 +55,7 @@ require (
 	github.com/yuin/goldmark v1.1.21
 	github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5
 	go.opencensus.io v0.22.0 // indirect
+	go.uber.org/atomic v1.4.0
 	gocloud.dev v0.15.0
 	golang.org/x/image v0.0.0-20191214001246-9130b4cfad52
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect

diff --git a/go.sum b/go.sum
@@ -421,6 +421,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=

diff --git a/hugolib/alias.go b/hugolib/alias.go
@@ -81,7 +81,7 @@ func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format
 }
 
 func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p page.Page) (err error) {
-	handler := newAliasHandler(s.Tmpl, s.Log, allowRoot)
+	handler := newAliasHandler(s.Tmpl(), s.Log, allowRoot)
 
 	s.Log.DEBUG.Println("creating alias:", path, "redirecting to", permalink)
 

diff --git a/hugolib/cascade_test.go b/hugolib/cascade_test.go
@@ -68,12 +68,12 @@ func TestCascade(t *testing.T) {
         42|taxonomy|tags/blue|blue|home.png|tags|HTML-|
         42|section|sect3|Cascade Home|home.png|sect3|HTML-|
         42|taxonomyTerm|tags|Cascade Home|home.png|tags|HTML-|
-        42|page|p2.md|Cascade Home|home.png|page|HTML-|
+        42|page|p2.md|Cascade Home|home.png||HTML-|
         42|page|sect2/p2.md|Cascade Home|home.png|sect2|HTML-|
         42|page|sect3/p1.md|Cascade Home|home.png|sect3|HTML-|
         42|taxonomy|tags/green|green|home.png|tags|HTML-|
-        42|home|_index.md|Home|home.png|page|HTML-|
-        42|page|p1.md|p1|home.png|page|HTML-|
+        42|home|_index.md|Home|home.png||HTML-|
+        42|page|p1.md|p1|home.png||HTML-|
         42|section|sect1/_index.md|Sect1|sect1.png|stype|HTML-|
         42|section|sect1/s1_2/_index.md|Sect1_2|sect1.png|stype|HTML-|
         42|page|sect1/s1_2/p1.md|Sect1_2_p1|sect1.png|stype|HTML-|

diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go
@@ -13,7 +13,9 @@
 
 package hugolib
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestRenderHooks(t *testing.T) {
 	config := `

diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go
@@ -340,6 +340,8 @@ b = "B param"
 }
 
 func TestModulesIncompatible(t *testing.T) {
+	t.Parallel()
+
 	b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", `
 baseURL="https://example.org"
 
@@ -518,6 +520,7 @@ weight = 2
 }
 
 func TestMountsProject(t *testing.T) {
+	t.Parallel()
 
 	config := `
 
@@ -547,6 +550,7 @@ title: "My Page"
 
 // https://github.com/gohugoio/hugo/issues/6684
 func TestMountsContentFile(t *testing.T) {
+	t.Parallel()
 	c := qt.New(t)
 	workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-content-file")
 	c.Assert(err, qt.IsNil)

diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
@@ -121,6 +121,9 @@ type hugoSitesInit struct {
 	// Loads the data from all of the /data folders.
 	data *lazy.Init
 
+	// Performs late initialization (before render) of the templates.
+	layouts *lazy.Init
+
 	// Loads the Git info for all the pages if enabled.
 	gitInfo *lazy.Init
 
@@ -130,6 +133,7 @@ type hugoSitesInit struct {
 
 func (h *hugoSitesInit) Reset() {
 	h.data.Reset()
+	h.layouts.Reset()
 	h.gitInfo.Reset()
 	h.translations.Reset()
 }
@@ -271,6 +275,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		Sites:        sites,
 		init: &hugoSitesInit{
 			data:         lazy.New(),
+			layouts:      lazy.New(),
 			gitInfo:      lazy.New(),
 			translations: lazy.New(),
 		},
@@ -289,6 +294,15 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		return nil, nil
 	})
 
+	h.init.layouts.Add(func() (interface{}, error) {
+		for _, s := range h.Sites {
+			if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil {
+				return nil, err
+			}
+		}
+		return nil, nil
+	})
+
 	h.init.translations.Add(func() (interface{}, error) {
 		if len(h.Sites) > 1 {
 			allTranslations := pagesToTranslationsMap(h.Sites)
@@ -429,10 +443,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 
 func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateManager) error) func(templ tpl.TemplateManager) error {
 	return func(templ tpl.TemplateManager) error {
-		if err := templ.LoadTemplates(""); err != nil {
-			return err
-		}
-
 		for _, wt := range withTemplates {
 			if wt == nil {
 				continue
@@ -619,10 +629,10 @@ func (h *HugoSites) renderCrossSitesArtifacts() error {
 
 	s := h.Sites[0]
 
-	smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"}
+	templ := s.lookupLayouts("sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml")
 
 	return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex",
-		s.siteCfg.sitemap.Filename, h.toSiteInfos(), smLayouts...)
+		s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ)
 }
 
 func (h *HugoSites) removePageByFilename(filename string) {
@@ -832,7 +842,7 @@ func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) {
 				if po.cp == nil {
 					continue
 				}
-				for id, _ := range idset {
+				for id := range idset {
 					if po.cp.dependencyTracker.Search(id) != nil {
 						po.cp.Reset()
 						continue OUTPUTS
@@ -841,7 +851,7 @@ func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) {
 			}
 
 			for _, s := range p.shortcodeState.shortcodes {
-				for id, _ := range idset {
+				for id := range idset {
 					if idm, ok := s.info.(identity.Manager); ok && idm.Search(id) != nil {
 						for _, po := range p.pageOutputs {
 							if po.cp != nil {