Add page fragments support to Related

The main topic of this commit is that you can now index fragments (content heading identifiers) when calling `.Related`.

You can do this by:

* Configure one or more indices with type `fragments`
* The name of those index configurations maps to an (optional) front matter slice with fragment references. This allows you to link
page<->fragment and page<->page.
* This also will index all the fragments (heading identifiers) of the pages.

It's also possible to use type `fragments` indices in shortcode, e.g.:

```
{{ $related := site.RegularPages.Related .Page }}
```

But, and this is important, you need to include the shortcode using the `{{<` delimiter. Not doing so will create infinite loops and timeouts.

This commit also:

* Consolidates all `.Related*` methods into one, which takes either a `Page` or an options map as its only argument.
* Add `context.Context` to all of the content related Page API. Turns out it wasn't strictly needed for this particular feature, but it will
soon become usefil, e.g. in gohugoio#9339.

Closes gohugoio#10711
Updates gohugoio#9339
Updates gohugoio#10725

common/collections/slice.go

@@ -15,6 +15,7 @@ package collections
 
 import (
 	"reflect"
+	"sort"
 )
 
 // Slicer defines a very generic way to create a typed slice. This is used
@@ -74,3 +75,22 @@ func StringSliceToInterfaceSlice(ss []string) []any {
 	return result
 
 }
+
+type SortedStringSlice []string
+
+// Contains returns true if s is in ss.
+func (ss SortedStringSlice) Contains(s string) bool {
+	i := sort.SearchStrings(ss, s)
+	return i < len(ss) && ss[i] == s
+}
+
+// Count returns the number of times s is in ss.
+func (ss SortedStringSlice) Count(s string) int {
+	var count int
+	i := sort.SearchStrings(ss, s)
+	for i < len(ss) && ss[i] == s {
+		count++
+		i++
+	}
+	return count
+}

common/collections/slice_test.go

@@ -122,3 +122,18 @@ func TestSlice(t *testing.T) {
 		c.Assert(test.expected, qt.DeepEquals, result, errMsg)
 	}
 }
+
+func TestSortedStringSlice(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	var s SortedStringSlice = []string{"a", "b", "b", "b", "c", "d"}
+
+	c.Assert(s.Contains("a"), qt.IsTrue)
+	c.Assert(s.Contains("b"), qt.IsTrue)
+	c.Assert(s.Contains("z"), qt.IsFalse)
+	c.Assert(s.Count("b"), qt.Equals, 3)
+	c.Assert(s.Count("z"), qt.Equals, 0)
+	c.Assert(s.Count("a"), qt.Equals, 1)
+
+}

go.mod

@@ -61,6 +61,7 @@ require (
 	github.com/yuin/goldmark v1.5.4
 	go.uber.org/atomic v1.10.0
 	gocloud.dev v0.28.0
+	golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
 	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
 	golang.org/x/net v0.4.0
 	golang.org/x/sync v0.1.0

go.sum

@@ -2002,6 +2002,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

hugolib/content_factory.go

@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"path/filepath"
@@ -83,7 +84,7 @@ func (f ContentFactory) ApplyArchetypeTemplate(w io.Writer, p page.Page, archety
 		return fmt.Errorf("failed to parse archetype template: %s: %w", err, err)
 	}
 
-	result, err := executeToString(ps.s.Tmpl(), templ, d)
+	result, err := executeToString(context.TODO(), ps.s.Tmpl(), templ, d)
 	if err != nil {
 		return fmt.Errorf("failed to execute archetype template: %s: %w", err, err)
 	}

hugolib/content_map_page.go

@@ -171,7 +171,7 @@ func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapB
 		return nil, err
 	}
 
-	ps.init.Add(func() (any, error) {
+	ps.init.Add(func(context.Context) (any, error) {
 		pp, err := newPagePaths(s, ps, metaProvider)
 		if err != nil {
 			return nil, err

hugolib/embedded_shortcodes_test.go

@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"html/template"
@@ -70,7 +71,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
 
 	c.Assert(len(s.RegularPages()), qt.Equals, 1)
 
-	content, err := s.RegularPages()[0].Content()
+	content, err := s.RegularPages()[0].Content(context.Background())
 	c.Assert(err, qt.IsNil)
 	output := cast.ToString(content)
 

hugolib/hugo_sites.go

@@ -194,15 +194,15 @@ func (h *hugoSitesInit) Reset() {
 }
 
 func (h *HugoSites) Data() map[string]any {
-	if _, err := h.init.data.Do(); err != nil {
+	if _, err := h.init.data.Do(context.Background()); err != nil {
 		h.SendError(fmt.Errorf("failed to load data: %w", err))
 		return nil
 	}
 	return h.data
 }
 
 func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) {
-	if _, err := h.init.gitInfo.Do(); err != nil {
+	if _, err := h.init.gitInfo.Do(context.Background()); err != nil {
 		return source.GitInfo{}, err
 	}
 
@@ -214,7 +214,7 @@ func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) {
 }
 
 func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) {
-	if _, err := h.init.gitInfo.Do(); err != nil {
+	if _, err := h.init.gitInfo.Do(context.Background()); err != nil {
 		return nil, err
 	}
 
@@ -363,15 +363,15 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		donec: make(chan bool),
 	}
 
-	h.init.data.Add(func() (any, error) {
+	h.init.data.Add(func(context.Context) (any, error) {
 		err := h.loadData(h.PathSpec.BaseFs.Data.Dirs)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load data: %w", err)
 		}
 		return nil, nil
 	})
 
-	h.init.layouts.Add(func() (any, error) {
+	h.init.layouts.Add(func(context.Context) (any, error) {
 		for _, s := range h.Sites {
 			if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil {
 				return nil, err
@@ -380,7 +380,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		return nil, nil
 	})
 
-	h.init.translations.Add(func() (any, error) {
+	h.init.translations.Add(func(context.Context) (any, error) {
 		if len(h.Sites) > 1 {
 			allTranslations := pagesToTranslationsMap(h.Sites)
 			assignTranslationsToPages(allTranslations, h.Sites)
@@ -389,7 +389,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		return nil, nil
 	})
 
-	h.init.gitInfo.Add(func() (any, error) {
+	h.init.gitInfo.Add(func(context.Context) (any, error) {
 		err := h.loadGitInfo()
 		if err != nil {
 			return nil, fmt.Errorf("failed to load Git info: %w", err)

hugolib/hugo_sites_build.go

@@ -268,7 +268,7 @@ func (h *HugoSites) assemble(bcfg *BuildCfg) error {
 }
 
 func (h *HugoSites) render(config *BuildCfg) error {
-	if _, err := h.init.layouts.Do(); err != nil {
+	if _, err := h.init.layouts.Do(context.Background()); err != nil {
 		return err
 	}
 

hugolib/hugo_sites_build_errors_test.go

@@ -396,7 +396,7 @@ line 4
 
 }
 
-func TestErrorNestedShortocde(t *testing.T) {
+func TestErrorNestedShortcode(t *testing.T) {
 	t.Parallel()
 
 	files := `

hugolib/image_test.go

@@ -14,162 +14,9 @@
 package hugolib
 
 import (
-	"io"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
 	"testing"
-
-	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/htesting"
-
-	qt "github.com/frankban/quicktest"
-	"github.com/gohugoio/hugo/hugofs"
 )
 
-// We have many tests for the different resize operations etc. in the resource package,
-// this is an integration test.
-func TestImageOps(t *testing.T) {
-	c := qt.New(t)
-	// Make this a real as possible.
-	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "image-resize")
-	c.Assert(err, qt.IsNil)
-	defer clean()
-
-	newBuilder := func(timeout any) *sitesBuilder {
-		v := config.NewWithTestDefaults()
-		v.Set("workingDir", workDir)
-		v.Set("baseURL", "https://example.org")
-		v.Set("timeout", timeout)
-
-		b := newTestSitesBuilder(t).WithWorkingDir(workDir)
-		b.Fs = hugofs.NewDefault(v)
-		b.WithViper(v)
-		b.WithContent("mybundle/index.md", `
----
-title: "My bundle"
----
-
-{{< imgproc >}}
-
-`)
-
-		b.WithTemplatesAdded(
-			"shortcodes/imgproc.html", `
-{{ $img := resources.Get "images/sunset.jpg" }}
-{{ $r := $img.Resize "129x239" }}
-IMG SHORTCODE: {{ $r.RelPermalink }}/{{ $r.Width }}
-`,
-			"index.html", `
-{{ $p := .Site.GetPage "mybundle" }}
-{{ $img1 := resources.Get "images/sunset.jpg" }}
-{{ $img2 := $p.Resources.GetMatch "sunset.jpg" }}
-{{ $img3 := resources.GetMatch "images/*.jpg" }}
-{{ $r := $img1.Resize "123x234" }}
-{{ $r2 := $r.Resize "12x23" }}
-{{ $b := $img2.Resize "345x678" }}
-{{ $b2 := $b.Resize "34x67" }}
-{{ $c := $img3.Resize "456x789" }}
-{{ $fingerprinted := $img1.Resize "350x" | fingerprint }}
-
-{{ $images := slice $r $r2 $b $b2 $c $fingerprinted }}
-
-{{ range $i, $r := $images }}
-{{ printf "Resized%d:" (add $i  1) }} {{ $r.Name }}|{{ $r.Width }}|{{ $r.Height }}|{{ $r.MediaType }}|{{ $r.RelPermalink }}|
-{{ end }}
-
-{{ $blurryGrayscale1 := $r | images.Filter images.Grayscale (images.GaussianBlur 8) }}
-BG1: {{ $blurryGrayscale1.RelPermalink }}/{{ $blurryGrayscale1.Width }}
-{{ $blurryGrayscale2 := $r.Filter images.Grayscale (images.GaussianBlur 8) }}
-BG2: {{ $blurryGrayscale2.RelPermalink }}/{{ $blurryGrayscale2.Width }}
-{{ $blurryGrayscale2_2 := $r.Filter images.Grayscale (images.GaussianBlur 8) }}
-BG2_2: {{ $blurryGrayscale2_2.RelPermalink }}/{{ $blurryGrayscale2_2.Width }}
-
-{{ $filters := slice images.Grayscale (images.GaussianBlur 9) }}
-{{ $blurryGrayscale3 := $r | images.Filter $filters }}
-BG3: {{ $blurryGrayscale3.RelPermalink }}/{{ $blurryGrayscale3.Width }}
-
-{{ $blurryGrayscale4 := $r.Filter $filters }}
-BG4: {{ $blurryGrayscale4.RelPermalink }}/{{ $blurryGrayscale4.Width }}
-
-{{ $p.Content }}
-
-`)
-
-		return b
-	}
-
-	imageDir := filepath.Join(workDir, "assets", "images")
-	bundleDir := filepath.Join(workDir, "content", "mybundle")
-
-	c.Assert(os.MkdirAll(imageDir, 0777), qt.IsNil)
-	c.Assert(os.MkdirAll(bundleDir, 0777), qt.IsNil)
-	src, err := os.Open("testdata/sunset.jpg")
-	c.Assert(err, qt.IsNil)
-	out, err := os.Create(filepath.Join(imageDir, "sunset.jpg"))
-	c.Assert(err, qt.IsNil)
-	_, err = io.Copy(out, src)
-	c.Assert(err, qt.IsNil)
-	out.Close()
-
-	src.Seek(0, 0)
-
-	out, err = os.Create(filepath.Join(bundleDir, "sunset.jpg"))
-	c.Assert(err, qt.IsNil)
-	_, err = io.Copy(out, src)
-	c.Assert(err, qt.IsNil)
-	out.Close()
-	src.Close()
-
-	// First build it with a very short timeout to trigger errors.
-	b := newBuilder("10ns")
-
-	imgExpect := `
-Resized1: images/sunset.jpg|123|234|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_123x234_resize_q75_box.jpg|
-Resized2: images/sunset.jpg|12|23|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ada4bb1a57f77a63306e3bd67286248e.jpg|
-Resized3: sunset.jpg|345|678|image/jpeg|/mybundle/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_345x678_resize_q75_box.jpg|
-Resized4: sunset.jpg|34|67|image/jpeg|/mybundle/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_44d8c928664d7c5a67377c6ec58425ce.jpg|
-Resized5: images/sunset.jpg|456|789|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_456x789_resize_q75_box.jpg|
-Resized6: images/sunset.jpg|350|219|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_350x0_resize_q75_box.a86fe88d894e5db613f6aa8a80538fefc25b20fa24ba0d782c057adcef616f56.jpg|
-BG1: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_2ae8bb993431ec1aec40fe59927b46b4.jpg/123
-BG2: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_2ae8bb993431ec1aec40fe59927b46b4.jpg/123
-BG3: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg/123
-BG4: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg/123
-IMG SHORTCODE: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_129x239_resize_q75_box.jpg/129
-`
-
-	assertImages := func() {
-		b.Helper()
-		b.AssertFileContent("public/index.html", imgExpect)
-		b.AssertImage(350, 219, "public/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_350x0_resize_q75_box.a86fe88d894e5db613f6aa8a80538fefc25b20fa24ba0d782c057adcef616f56.jpg")
-		b.AssertImage(129, 239, "public/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_129x239_resize_q75_box.jpg")
-	}
-
-	err = b.BuildE(BuildCfg{})
-	if runtime.GOOS != "windows" && !strings.Contains(runtime.GOARCH, "arm") && !htesting.IsGitHubAction() {
-		// TODO(bep)
-		c.Assert(err, qt.Not(qt.IsNil))
-	}
-
-	b = newBuilder(29000)
-	b.Build(BuildCfg{})
-
-	assertImages()
-
-	// Truncate one image.
-	imgInCache := filepath.Join(workDir, "resources/_gen/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg")
-	f, err := os.Create(imgInCache)
-	c.Assert(err, qt.IsNil)
-	f.Close()
-
-	// Build it again to make sure we read images from file cache.
-	b = newBuilder("30s")
-	b.Build(BuildCfg{})
-
-	assertImages()
-}
-
 func TestImageResizeMultilingual(t *testing.T) {
 	b := newTestSitesBuilder(t).WithConfigFile("toml", `
 baseURL="https://example.org"

hugolib/language_content_dir_test.go

@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -245,7 +246,7 @@ Content.
 	c.Assert(svP2.Language().Lang, qt.Equals, "sv")
 	c.Assert(nnP2.Language().Lang, qt.Equals, "nn")
 
-	content, _ := nnP2.Content()
+	content, _ := nnP2.Content(context.Background())
 	contentStr := cast.ToString(content)
 	c.Assert(contentStr, qt.Contains, "SVP3-REF: https://example.org/sv/sect/p-sv-3/")
 	c.Assert(contentStr, qt.Contains, "SVP3-RELREF: /sv/sect/p-sv-3/")