Add a way to merge pages by language

As an example:

```html
{{ $pages := .Site.RegularPages | lang.Merge $frSite.RegularPages | lang.Merge $enSite.RegularPages }}
```

Will "fill in the gaps" in the current site with, from left to right, content from the French site, and lastly the English.

Fixes #4463

hugolib/hugo_sites_build.go

@@ -15,9 +15,12 @@ package hugolib
 
 import (
 	"bytes"
+	"fmt"
 
 	"errors"
 
+	jww "github.com/spf13/jwalterweatherman"
+
 	"github.com/fsnotify/fsnotify"
 	"github.com/gohugoio/hugo/helpers"
 )
@@ -71,6 +74,11 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 		h.Log.FEEDBACK.Println()
 	}
 
+	errorCount := h.Log.LogCountForLevel(jww.LevelError)
+	if errorCount > 0 {
+		return fmt.Errorf("logged %d error(s)", errorCount)
+	}
+
 	return nil
 
 }

hugolib/hugo_sites_build_test.go

@@ -1069,6 +1069,7 @@ func createMultiTestSitesForConfig(t *testing.T, siteConfig testSiteConfig, conf
 
 	mf := siteConfig.Fs
 
+	// TODO(bep) cleanup/remove duplication, use the new testBuilder in testhelpers_test
 	// Add some layouts
 	if err := afero.WriteFile(mf,
 		filepath.Join("layouts", "_default/single.html"),
@@ -1368,7 +1369,7 @@ func readSource(t *testing.T, fs *hugofs.Fs, filename string) string {
 }
 
 func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
-	filename = filepath.FromSlash(filename)
+	filename = filepath.Clean(filename)
 	b, err := afero.ReadFile(fs, filename)
 	if err != nil {
 		// Print some debug info

hugolib/page.go

@@ -263,6 +263,16 @@ type Page struct {
 	targetPathDescriptorPrototype *targetPathDescriptor
 }
 
+// Sites is a convenience method to get all the Hugo sites/languages configured.
+func (p *Page) Sites() SiteInfos {
+	infos := make(SiteInfos, len(p.s.owner.Sites))
+	for i, site := range p.s.owner.Sites {
+		infos[i] = &site.Info
+	}
+
+	return infos
+}
+
 // SearchKeywords implements the related.Document interface needed for fast page searches.
 func (p *Page) SearchKeywords(cfg related.IndexConfig) ([]related.Keyword, error) {
 

hugolib/pageCache.go

@@ -17,31 +17,59 @@ import (
 	"sync"
 )
 
+type pageCacheEntry struct {
+	in  []Pages
+	out Pages
+}
+
+func (entry pageCacheEntry) matches(pageLists []Pages) bool {
+	if len(entry.in) != len(pageLists) {
+		return false
+	}
+	for i, p := range pageLists {
+		if !fastEqualPages(p, entry.in[i]) {
+			return false
+		}
+	}
+
+	return true
+}
+
 type pageCache struct {
 	sync.RWMutex
-	m map[string][][2]Pages
+	m map[string][]pageCacheEntry
 }
 
 func newPageCache() *pageCache {
-	return &pageCache{m: make(map[string][][2]Pages)}
+	return &pageCache{m: make(map[string][]pageCacheEntry)}
 }
 
-// get gets a Pages slice from the cache matching the given key and Pages slice.
-// If none found in cache, a copy of the supplied slice is created.
+// get/getP gets a Pages slice from the cache matching the given key and
+// all the provided Pages slices.
+// If none found in cache, a copy of the first slice is created.
 //
 // If an apply func is provided, that func is applied to the newly created copy.
 //
+// The getP variant' apply func takes a pointer to Pages.
+//
 // The cache and the execution of the apply func is protected by a RWMutex.
-func (c *pageCache) get(key string, p Pages, apply func(p Pages)) (Pages, bool) {
+func (c *pageCache) get(key string, apply func(p Pages), pageLists ...Pages) (Pages, bool) {
+	return c.getP(key, func(p *Pages) {
+		if apply != nil {
+			apply(*p)
+		}
+	}, pageLists...)
+}
+
+func (c *pageCache) getP(key string, apply func(p *Pages), pageLists ...Pages) (Pages, bool) {
 	c.RLock()
 	if cached, ok := c.m[key]; ok {
-		for _, ps := range cached {
-			if fastEqualPages(p, ps[0]) {
+		for _, entry := range cached {
+			if entry.matches(pageLists) {
 				c.RUnlock()
-				return ps[1], true
+				return entry.out, true
 			}
 		}
-
 	}
 	c.RUnlock()
 
@@ -50,23 +78,25 @@ func (c *pageCache) get(key string, p Pages, apply func(p Pages)) (Pages, bool)
 
 	// double-check
 	if cached, ok := c.m[key]; ok {
-		for _, ps := range cached {
-			if fastEqualPages(p, ps[0]) {
-				return ps[1], true
+		for _, entry := range cached {
+			if entry.matches(pageLists) {
+				return entry.out, true
 			}
 		}
 	}
 
+	p := pageLists[0]
 	pagesCopy := append(Pages(nil), p...)
 
 	if apply != nil {
-		apply(pagesCopy)
+		apply(&pagesCopy)
 	}
 
+	entry := pageCacheEntry{in: pageLists, out: pagesCopy}
 	if v, ok := c.m[key]; ok {
-		c.m[key] = append(v, [2]Pages{p, pagesCopy})
+		c.m[key] = append(v, entry)
 	} else {
-		c.m[key] = [][2]Pages{{p, pagesCopy}}
+		c.m[key] = []pageCacheEntry{entry}
 	}
 
 	return pagesCopy, false

hugolib/pageCache_test.go

@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -51,17 +52,17 @@ func TestPageCache(t *testing.T) {
 			defer wg.Done()
 			for k, pages := range testPageSets {
 				l1.Lock()
-				p, c := c1.get("k1", pages, nil)
+				p, c := c1.get("k1", nil, pages)
 				assert.Equal(t, !atomic.CompareAndSwapUint64(&o1, uint64(k), uint64(k+1)), c)
 				l1.Unlock()
-				p2, c2 := c1.get("k1", p, nil)
+				p2, c2 := c1.get("k1", nil, p)
 				assert.True(t, c2)
 				assert.True(t, fastEqualPages(p, p2))
 				assert.True(t, fastEqualPages(p, pages))
 				assert.NotNil(t, p)
 
 				l2.Lock()
-				p3, c3 := c1.get("k2", pages, changeFirst)
+				p3, c3 := c1.get("k2", changeFirst, pages)
 				assert.Equal(t, !atomic.CompareAndSwapUint64(&o2, uint64(k), uint64(k+1)), c3)
 				l2.Unlock()
 				assert.NotNil(t, p3)
@@ -71,3 +72,17 @@ func TestPageCache(t *testing.T) {
 	}
 	wg.Wait()
 }
+
+func BenchmarkPageCache(b *testing.B) {
+	cache := newPageCache()
+	pages := make(Pages, 30)
+	for i := 0; i < 30; i++ {
+		pages[i] = &Page{title: "p" + strconv.Itoa(i)}
+	}
+	key := "key"
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		cache.getP(key, nil, pages)
+	}
+}

hugolib/pageSort.go

@@ -14,8 +14,9 @@
 package hugolib
 
 import (
-	"github.com/spf13/cast"
 	"sort"
+
+	"github.com/spf13/cast"
 )
 
 var spc = newPageCache()
@@ -115,7 +116,7 @@ func (p Pages) Limit(n int) Pages {
 // This may safely be executed  in parallel.
 func (p Pages) ByWeight() Pages {
 	key := "pageSort.ByWeight"
-	pages, _ := spc.get(key, p, pageBy(defaultPageSort).Sort)
+	pages, _ := spc.get(key, pageBy(defaultPageSort).Sort, p)
 	return pages
 }
 
@@ -132,7 +133,7 @@ func (p Pages) ByTitle() Pages {
 		return p1.title < p2.title
 	}
 
-	pages, _ := spc.get(key, p, pageBy(title).Sort)
+	pages, _ := spc.get(key, pageBy(title).Sort, p)
 	return pages
 }
 
@@ -149,7 +150,7 @@ func (p Pages) ByLinkTitle() Pages {
 		return p1.linkTitle < p2.linkTitle
 	}
 
-	pages, _ := spc.get(key, p, pageBy(linkTitle).Sort)
+	pages, _ := spc.get(key, pageBy(linkTitle).Sort, p)
 
 	return pages
 }
@@ -167,7 +168,7 @@ func (p Pages) ByDate() Pages {
 		return p1.Date.Unix() < p2.Date.Unix()
 	}
 
-	pages, _ := spc.get(key, p, pageBy(date).Sort)
+	pages, _ := spc.get(key, pageBy(date).Sort, p)
 
 	return pages
 }
@@ -185,7 +186,7 @@ func (p Pages) ByPublishDate() Pages {
 		return p1.PublishDate.Unix() < p2.PublishDate.Unix()
 	}
 
-	pages, _ := spc.get(key, p, pageBy(pubDate).Sort)
+	pages, _ := spc.get(key, pageBy(pubDate).Sort, p)
 
 	return pages
 }
@@ -203,7 +204,7 @@ func (p Pages) ByExpiryDate() Pages {
 		return p1.ExpiryDate.Unix() < p2.ExpiryDate.Unix()
 	}
 
-	pages, _ := spc.get(key, p, pageBy(expDate).Sort)
+	pages, _ := spc.get(key, pageBy(expDate).Sort, p)
 
 	return pages
 }
@@ -221,7 +222,7 @@ func (p Pages) ByLastmod() Pages {
 		return p1.Lastmod.Unix() < p2.Lastmod.Unix()
 	}
 
-	pages, _ := spc.get(key, p, pageBy(date).Sort)
+	pages, _ := spc.get(key, pageBy(date).Sort, p)
 
 	return pages
 }
@@ -239,7 +240,7 @@ func (p Pages) ByLength() Pages {
 		return len(p1.Content) < len(p2.Content)
 	}
 
-	pages, _ := spc.get(key, p, pageBy(length).Sort)
+	pages, _ := spc.get(key, pageBy(length).Sort, p)
 
 	return pages
 }
@@ -253,7 +254,7 @@ func (p Pages) ByLanguage() Pages {
 
 	key := "pageSort.ByLanguage"
 
-	pages, _ := spc.get(key, p, pageBy(languagePageSort).Sort)
+	pages, _ := spc.get(key, pageBy(languagePageSort).Sort, p)
 
 	return pages
 }
@@ -272,7 +273,7 @@ func (p Pages) Reverse() Pages {
 		}
 	}
 
-	pages, _ := spc.get(key, p, reverseFunc)
+	pages, _ := spc.get(key, reverseFunc, p)
 
 	return pages
 }
@@ -297,7 +298,7 @@ func (p Pages) ByParam(paramsKey interface{}) Pages {
 		return s1 < s2
 	}
 
-	pages, _ := spc.get(key, p, pageBy(paramsKeyComparator).Sort)
+	pages, _ := spc.get(key, pageBy(paramsKeyComparator).Sort, p)
 
 	return pages
 }

hugolib/pages_language_merge.go

@@ -0,0 +1,61 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"fmt"
+)
+
+var (
+	_ pagesLanguageMerger = (*Pages)(nil)
+)
+
+type pagesLanguageMerger interface {
+	MergeByLanguage(other Pages) Pages
+	// Needed for integration with the tpl package.
+	MergeByLanguageInterface(other interface{}) (interface{}, error)
+}
+
+// MergeByLanguage supplies missing translations in p1 with values from p2.
+// The result is sorted by the default sort order for pages.
+func (p1 Pages) MergeByLanguage(p2 Pages) Pages {
+	merge := func(pages *Pages) {
+		m := make(map[string]bool)
+		for _, p := range *pages {
+			m[p.TranslationKey()] = true
+		}
+
+		for _, p := range p2 {
+			if _, found := m[p.TranslationKey()]; !found {
+				*pages = append(*pages, p)
+			}
+		}
+
+		pages.Sort()
+	}
+
+	out, _ := spc.getP("pages.MergeByLanguage", merge, p1, p2)
+
+	return out
+}
+
+// MergeByLanguageInterface is the generic version of MergeByLanguage. It
+// is here just so it can be called from the tpl package.
+func (p1 Pages) MergeByLanguageInterface(in interface{}) (interface{}, error) {
+	p2, ok := in.(Pages)
+	if !ok {
+		return nil, fmt.Errorf("%T cannot be merged by language", in)
+	}
+	return p1.MergeByLanguage(p2), nil
+}