Simplify page tree logic

This is preparation for gohugoio#6041.

For historic reasons, the code for bulding the section tree and the taxonomies were very much separate.

This works, but makes it hard to extend, maintain, and possibly not so fast as it could be.

This simplification also introduces 3 slightly breaking changes, which I suspect most people will be pleased about. See referenced issues:

Fixes gohugoio#6154
Fixes gohugoio#6153
Fixes gohugoio#6152

common/herrors/errors.go

@@ -50,9 +50,10 @@ func FprintStackTrace(w io.Writer, err error) {
 // Recover is a helper function that can be used to capture panics.
 // Put this at the top of a method/function that crashes in a template:
 //     defer herrors.Recover()
-func Recover() {
+func Recover(args ...interface{}) {
 	if r := recover(); r != nil {
-		fmt.Println("stacktrace from panic: \n" + string(debug.Stack()))
+		args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n")
+		fmt.Println(args...)
 	}
 
 }

common/maps/maps_get.go

@@ -0,0 +1,28 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"github.com/spf13/cast"
+)
+
+// GetString tries to get a value with key from map m and convert it to a string.
+// It will return an empty string if not found or if it cannot be convertd to a string.
+func GetString(m map[string]interface{}, key string) string {
+	v, found := m[key]
+	if !found {
+		return ""
+	}
+	return cast.ToString(v)
+}

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
 	github.com/alecthomas/chroma v0.6.4
 	github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 // indirect
+	github.com/armon/go-radix v1.0.0
 	github.com/aws/aws-sdk-go v1.19.40
 	github.com/bep/debounce v1.2.0
 	github.com/bep/gitmap v1.1.0

go.sum

@@ -58,6 +58,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
+github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=

hugolib/hugo_sites.go

@@ -14,9 +14,7 @@
 package hugolib
 
 import (
-	"fmt"
 	"io"
-	"path"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -623,142 +621,14 @@ func (h *HugoSites) renderCrossSitesArtifacts() error {
 		s.siteCfg.sitemap.Filename, h.toSiteInfos(), smLayouts...)
 }
 
-// createMissingPages creates home page, taxonomies etc. that isnt't created as an
-// effect of having a content file.
-func (h *HugoSites) createMissingPages() error {
-
-	for _, s := range h.Sites {
-		if s.isEnabled(page.KindHome) {
-			// home pages
-			homes := s.findWorkPagesByKind(page.KindHome)
-			if len(homes) > 1 {
-				panic("Too many homes")
-			}
-			var home *pageState
-			if len(homes) == 0 {
-				home = s.newPage(page.KindHome)
-				s.workAllPages = append(s.workAllPages, home)
-			} else {
-				home = homes[0]
-			}
-
-			s.home = home
-		}
-
-		// Will create content-less root sections.
-		newSections := s.assembleSections()
-		s.workAllPages = append(s.workAllPages, newSections...)
-
-		taxonomyTermEnabled := s.isEnabled(page.KindTaxonomyTerm)
-		taxonomyEnabled := s.isEnabled(page.KindTaxonomy)
-
-		// taxonomy list and terms pages
-		taxonomies := s.Language().GetStringMapString("taxonomies")
-		if len(taxonomies) > 0 {
-			taxonomyPages := s.findWorkPagesByKind(page.KindTaxonomy)
-			taxonomyTermsPages := s.findWorkPagesByKind(page.KindTaxonomyTerm)
-
-			// Make them navigable from WeightedPage etc.
-			for _, p := range taxonomyPages {
-				ni := p.getTaxonomyNodeInfo()
-				if ni == nil {
-					// This can be nil for taxonomies, e.g. an author,
-					// with a content file, but no actual usage.
-					// Create one.
-					sections := p.SectionsEntries()
-					if len(sections) < 2 {
-						// Invalid state
-						panic(fmt.Sprintf("invalid taxonomy state for %q with sections %v", p.pathOrTitle(), sections))
-					}
-					ni = p.s.taxonomyNodes.GetOrAdd(sections[0], path.Join(sections[1:]...))
-				}
-				ni.TransferValues(p)
-			}
-			for _, p := range taxonomyTermsPages {
-				p.getTaxonomyNodeInfo().TransferValues(p)
-			}
-
-			for _, plural := range taxonomies {
-				if taxonomyTermEnabled {
-					foundTaxonomyTermsPage := false
-					for _, p := range taxonomyTermsPages {
-						if p.SectionsPath() == plural {
-							foundTaxonomyTermsPage = true
-							break
-						}
-					}
-
-					if !foundTaxonomyTermsPage {
-						n := s.newPage(page.KindTaxonomyTerm, plural)
-						n.getTaxonomyNodeInfo().TransferValues(n)
-						s.workAllPages = append(s.workAllPages, n)
-					}
-				}
-
-				if taxonomyEnabled {
-					for termKey := range s.Taxonomies[plural] {
-
-						foundTaxonomyPage := false
-
-						for _, p := range taxonomyPages {
-							sectionsPath := p.SectionsPath()
-
-							if !strings.HasPrefix(sectionsPath, plural) {
-								continue
-							}
-
-							singularKey := strings.TrimPrefix(sectionsPath, plural)
-							singularKey = strings.TrimPrefix(singularKey, "/")
-
-							if singularKey == termKey {
-								foundTaxonomyPage = true
-								break
-							}
-						}
-
-						if !foundTaxonomyPage {
-							info := s.taxonomyNodes.Get(plural, termKey)
-							if info == nil {
-								panic("no info found")
-							}
-
-							n := s.newTaxonomyPage(info.term, info.plural, info.termKey)
-							info.TransferValues(n)
-							s.workAllPages = append(s.workAllPages, n)
-						}
-					}
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
 func (h *HugoSites) removePageByFilename(filename string) {
 	for _, s := range h.Sites {
 		s.removePageFilename(filename)
 	}
 }
 
+// TODO(bep) cm
 func (h *HugoSites) createPageCollections() error {
-	for _, s := range h.Sites {
-		for _, p := range s.rawAllPages {
-			if !s.isEnabled(p.Kind()) {
-				continue
-			}
-
-			shouldBuild := s.shouldBuild(p)
-			s.buildStats.update(p)
-			if shouldBuild {
-				if p.m.headless {
-					s.headlessPages = append(s.headlessPages, p)
-				} else {
-					s.workAllPages = append(s.workAllPages, p)
-				}
-			}
-		}
-	}
 
 	allPages := newLazyPagesFactory(func() page.Pages {
 		var pages page.Pages

hugolib/hugo_sites_build.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"runtime/trace"
-	"sort"
 
 	"github.com/gohugoio/hugo/output"
 
@@ -235,26 +234,41 @@ func (h *HugoSites) assemble(config *BuildCfg) error {
 		}
 	}
 
-	if err := h.createPageCollections(); err != nil {
-		return err
-	}
-
 	if config.whatChanged.source {
-		for _, s := range h.Sites {
-			if err := s.assembleTaxonomies(); err != nil {
-				return err
-			}
-		}
+		//for _, s := range h.Sites {
+		// TODO(bep) cm
+		//if err := s.assembleTaxonomies(); err != nil {
+		//	return err
+		//}
+		//}
 	}
 
-	// Create pagexs for the section pages etc. without content file.
-	if err := h.createMissingPages(); err != nil {
+	// Create pages for the section pages etc. without content file.
+	// TODO(bep) cm
+	/*if err := h.createMissingPages(); err != nil {
 		return err
-	}
+	}*/
 
 	for _, s := range h.Sites {
-		s.setupSitePages()
-		sort.Stable(s.workAllPages)
+		if err := s.createPagesMap(s); err != nil {
+			return err
+		}
+
+		if err := s.pagesMap.assembleTaxonomies(s); err != nil {
+			return err
+		}
+
+		if err := s.createWorkAllPages(); err != nil {
+			return err
+		}
+
+		// TODO(bep) cm
+		//s.setupSitePages()
+		//sort.Stable(s.workAllPages)
+	}
+
+	if err := h.createPageCollections(); err != nil {
+		return err
 	}
 
 	return nil

hugolib/hugo_sites_build_test.go

@@ -365,7 +365,6 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 	require.NotNil(t, enTags["tag1"])
 	require.NotNil(t, frTags["FRtag1"])
 	b.AssertFileContent("public/fr/plaques/FRtag1/index.html", "FRtag1|Bonjour|http://example.com/blog/fr/plaques/FRtag1/")
-	b.AssertFileContent("public/en/tags/tag1/index.html", "tag1|Hello|http://example.com/blog/en/tags/tag1/")
 
 	// Check Blackfriday config
 	require.True(t, strings.Contains(content(doc1fr), "«"), content(doc1fr))

hugolib/page.go

@@ -23,6 +23,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/maps"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 
 	"github.com/bep/gitmap"
@@ -121,6 +123,15 @@ func (p *pageState) MarshalJSON() ([]byte, error) {
 	return page.MarshalPageToJSON(p)
 }
 
+func (p *pageState) getPages() page.Pages {
+	b := p.bucket
+	if b == nil {
+		return nil
+	}
+	return b.pages
+
+}
+
 func (p *pageState) Pages() page.Pages {
 	p.pagesInit.Do(func() {
 		if p.pages != nil {
@@ -132,19 +143,18 @@ func (p *pageState) Pages() page.Pages {
 		switch p.Kind() {
 		case page.KindPage:
 		case page.KindHome:
+			// TODO(bep) cm
 			pages = p.s.RegularPages()
+		case page.KindSection:
+			pages = p.getPages()
 		case page.KindTaxonomy:
-			termInfo := p.getTaxonomyNodeInfo()
-			taxonomy := p.s.Taxonomies[termInfo.plural].Get(termInfo.termKey)
+			termInfo := p.bucket
+			plural := maps.GetString(termInfo.meta, "plural")
+			term := maps.GetString(termInfo.meta, "term")
+			taxonomy := p.s.Taxonomies[plural].Get(term)
 			pages = taxonomy.Pages()
 		case page.KindTaxonomyTerm:
-			plural := p.getTaxonomyNodeInfo().plural
-			// A list of all page.KindTaxonomy pages with matching plural
-			for _, p := range p.s.findPagesByKind(page.KindTaxonomy) {
-				if p.SectionsEntries()[0] == plural {
-					pages = append(pages, p)
-				}
-			}
+			pages = p.getPages()
 		case kind404, kindSitemap, kindRobotsTXT:
 			pages = p.s.Pages()
 		}
@@ -296,9 +306,9 @@ func (p *pageState) getLayoutDescriptor() output.LayoutDescriptor {
 				section = sections[0]
 			}
 		case page.KindTaxonomyTerm:
-			section = p.getTaxonomyNodeInfo().singular
+			// TODO(bep) cm section = p.getTaxonomyNodeInfo().singular
 		case page.KindTaxonomy:
-			section = p.getTaxonomyNodeInfo().parent.singular
+			// TODO(bep) cm section = p.getTaxonomyNodeInfo().parent.singular
 		default:
 		}
 
@@ -743,20 +753,6 @@ func (p *pageState) shiftToOutputFormat(isRenderingSite bool, idx int) error {
 	return nil
 }
 
-func (p *pageState) getTaxonomyNodeInfo() *taxonomyNodeInfo {
-	info := p.s.taxonomyNodes.Get(p.SectionsEntries()...)
-
-	if info == nil {
-		// There can be unused content pages for taxonomies (e.g. author that
-		// has not written anything, yet), and these will not have a taxonomy
-		// node created in the assemble taxonomies step.
-		return nil
-	}
-
-	return info
-
-}
-
 func (p *pageState) sortParentSections() {
 	if p.parent == nil {
 		return

hugolib/page__common.go

@@ -30,6 +30,8 @@ type pageCommon struct {
 	s *Site
 	m *pageMeta
 
+	bucket *pagesMapBucket
+
 	// Laziliy initialized dependencies.
 	init *lazy.Init