Add page fragments support to Related

The main topic of this commit is that you can now index fragments (content heading identifiers) when calling `.Related`.

You can do this by:

* Configure one or more indices with type `fragments`
* The name of those index configurations maps to an (optional) front matter slice with fragment references. This allows you to link
page<->fragment and page<->page.
* This also will index all the fragments (heading identifiers) of the pages.

It's also possible to use type `fragments` indices in shortcode, e.g.:

```
{{ $related := site.RegularPages.Related .Page }}
```

But, and this is important, you need to include the shortcode using the `{{<` delimiter. Not doing so will create infinite loops and timeouts.

This commit also:

* Adds two new methods to Page: Fragments (can also be used to build ToC) and HeadingsFiltered (this is only used in Related Content with
index type `fragments` and `enableFilter` set to true.
* Consolidates all `.Related*` methods into one, which takes either a `Page` or an options map as its only argument.
* Add `context.Context` to all of the content related Page API. Turns out it wasn't strictly needed for this particular feature, but it will
soon become usefil, e.g. in gohugoio#9339.

Closes gohugoio#10711
Updates gohugoio#9339
Updates gohugoio#10725

.gitignore

@@ -0,0 +1,2 @@
+
+*.test

common/collections/slice.go

@@ -15,6 +15,7 @@ package collections
 
 import (
 	"reflect"
+	"sort"
 )
 
 // Slicer defines a very generic way to create a typed slice. This is used
@@ -74,3 +75,22 @@ func StringSliceToInterfaceSlice(ss []string) []any {
 	return result
 
 }
+
+type SortedStringSlice []string
+
+// Contains returns true if s is in ss.
+func (ss SortedStringSlice) Contains(s string) bool {
+	i := sort.SearchStrings(ss, s)
+	return i < len(ss) && ss[i] == s
+}
+
+// Count returns the number of times s is in ss.
+func (ss SortedStringSlice) Count(s string) int {
+	var count int
+	i := sort.SearchStrings(ss, s)
+	for i < len(ss) && ss[i] == s {
+		count++
+		i++
+	}
+	return count
+}

common/collections/slice_test.go

@@ -122,3 +122,18 @@ func TestSlice(t *testing.T) {
 		c.Assert(test.expected, qt.DeepEquals, result, errMsg)
 	}
 }
+
+func TestSortedStringSlice(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	var s SortedStringSlice = []string{"a", "b", "b", "b", "c", "d"}
+
+	c.Assert(s.Contains("a"), qt.IsTrue)
+	c.Assert(s.Contains("b"), qt.IsTrue)
+	c.Assert(s.Contains("z"), qt.IsFalse)
+	c.Assert(s.Count("b"), qt.Equals, 3)
+	c.Assert(s.Count("z"), qt.Equals, 0)
+	c.Assert(s.Count("a"), qt.Equals, 1)
+
+}

docs/content/en/content-management/related.md

@@ -31,40 +31,82 @@ To list up to 5 related pages (which share the same _date_ or _keyword_ paramete
 {{ end }}
 {{< /code >}}
 
-### Methods
+The `Related` method takes one argument which may be a `Page` or a options map. The options map have these options:
 
-Here is the list of "Related" methods available on a page collection such `.RegularPages`.
+indices
+: The indices to search in.
 
-#### .Related PAGE
+document
+: The document to search for related content for.
 
-Returns a collection of pages related the given one.
+namedSlices
+: The keywords to search for.
+
+fragments
+: Fragments holds a a list of special keywords that is used for indices configured as type "fragments". This will match the fragment identifiers of the documents.
+
+A fictional example using all of the above options:
 
 ```go-html-template
-{{ $related := site.RegularPages.Related . }}
+{{ $page := . }}
+{{ $opts := 
+  "indices" (slice "tags" "keywords")
+  "document" $page
+  "namedSlices" (slice (keyVals "tags" "hugo" "rocks") (keyVals "date" $page.Date))
+  "fragments" (slice "heading-1" "heading-2")
+}}
 ```
 
-#### .RelatedIndices PAGE INDICE1 [INDICE2 ...]
+{{% note %}}
+We improved and simplified this feature in Hugo 0.111.0. Before this we had 3 different methods: `Related`, `RelatedTo` and `RelatedIndicies`. Now we have only one method: `Related`. The old methods are still available but deprecated. Also see [this blog article](https://regisphilibert.com/blog/2018/04/hugo-optmized-relashionships-with-related-content/) for a great explanation of more advanced usage of this feature.
+{{% /note %}}
+
+## Index Content Headings in Related Content
 
-Returns a collection of pages related to a given one restricted to a list of indices.
+{{< new-in "0.111.0" >}}
 
-```go-html-template
-{{ $related := site.RegularPages.RelatedIndices . "tags" "date" }}
-```
+Hugo can index the headings in your content and use this to find related content. You can enable this by adding a index of type `fragments` to your `related` configuration:
 
-#### .RelatedTo KEYVALS [KEYVALS2 ...]
 
-Returns a collection of pages related together by a set of indices and their match.
+```toml
+[related]
+threshold    = 20
+includeNewer = true
+toLower      = false
+[[related.indices]]
+name        = "fragmentrefs"
+type        = "fragments"
+applyFilter = false
+weight      = 80
+```
 
-In order to build those set and pass them as argument, one must use the `keyVals` function where the first argument would be the `indice` and the consecutive ones its potential `matches`.
+* The `name` maps to a optional front matter slice attribute that can be used to link from the page level down to the fragment/heading level.
+* If `applyFilter`is enabled, the `.HeadingsFiltered` on each page in the result will reflect the filtered headings. This is useful if you want to show the headings in the related content listing:
 
 ```go-html-template
-{{ $related := site.RegularPages.RelatedTo ( keyVals "tags" "hugo" "rocks")  ( keyVals "date" .Date ) }}
+{{ $related := .Site.RegularPages.Related . | first 5 }}
+{{ with $related }}
+  <h2>See Also</h2>
+  <ul>
+    {{ range . }}
+      <li>
+        <a href="{{ .RelPermalink }}">{{ .Title }}</a>
+        {{ with .HeadingsFiltered }}
+          <ul>
+            {{ range . }}
+              {{ $link := printf "%s#%s" $.RelPermalink .ID }}
+              <li>
+                <a href="{{ $link }}">{{ .Title }}</a>
+              </li>
+            {{ end }}
+          </ul>
+        {{ end }}
+      </li>
+    {{ end }}
+  </ul>
+{{ end }}
 ```
 
-{{% note %}}
-Read [this blog article](https://regisphilibert.com/blog/2018/04/hugo-optmized-relashionships-with-related-content/) for a great explanation of more advanced usage of this feature.
-{{% /note %}}
-
 ## Configure Related Content
 
 Hugo provides a sensible default configuration of Related Content, but you can fine-tune this in your configuration, on the global or language level if needed.
@@ -109,6 +151,12 @@ toLower
 name
 :  The index name. This value maps directly to a page param. Hugo supports string values (`author` in the example) and lists (`tags`, `keywords` etc.) and time and date objects.
 
+type
+: {{< new-in "0.111.0" >}}. One of `basic`(default) or `fragments`.
+
+applyFilter
+: {{< new-in "0.111.0" >}}. Apply a `type` specific filter to the result of a search. This is currently only used for the `fragments` type.
+
 weight
 : An integer weight that indicates _how important_ this parameter is relative to the other parameters.  It can be 0, which has the effect of turning this index off, or even negative. Test with different values to see what fits your content best.
 

go.mod

@@ -61,6 +61,7 @@ require (
 	github.com/yuin/goldmark v1.5.4
 	go.uber.org/atomic v1.10.0
 	gocloud.dev v0.28.0
+	golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
 	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
 	golang.org/x/net v0.4.0
 	golang.org/x/sync v0.1.0

go.sum

@@ -2002,6 +2002,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

hugolib/content_factory.go

@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"path/filepath"
@@ -83,7 +84,7 @@ func (f ContentFactory) ApplyArchetypeTemplate(w io.Writer, p page.Page, archety
 		return fmt.Errorf("failed to parse archetype template: %s: %w", err, err)
 	}
 
-	result, err := executeToString(ps.s.Tmpl(), templ, d)
+	result, err := executeToString(context.TODO(), ps.s.Tmpl(), templ, d)
 	if err != nil {
 		return fmt.Errorf("failed to execute archetype template: %s: %w", err, err)
 	}

hugolib/content_map_page.go

@@ -171,7 +171,7 @@ func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapB
 		return nil, err
 	}
 
-	ps.init.Add(func() (any, error) {
+	ps.init.Add(func(context.Context) (any, error) {
 		pp, err := newPagePaths(s, ps, metaProvider)
 		if err != nil {
 			return nil, err

hugolib/embedded_shortcodes_test.go

@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"html/template"
@@ -70,7 +71,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
 
 	c.Assert(len(s.RegularPages()), qt.Equals, 1)
 
-	content, err := s.RegularPages()[0].Content()
+	content, err := s.RegularPages()[0].Content(context.Background())
 	c.Assert(err, qt.IsNil)
 	output := cast.ToString(content)
 

hugolib/hugo_sites.go

@@ -194,15 +194,15 @@ func (h *hugoSitesInit) Reset() {
 }
 
 func (h *HugoSites) Data() map[string]any {
-	if _, err := h.init.data.Do(); err != nil {
+	if _, err := h.init.data.Do(context.Background()); err != nil {
 		h.SendError(fmt.Errorf("failed to load data: %w", err))
 		return nil
 	}
 	return h.data
 }
 
 func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) {
-	if _, err := h.init.gitInfo.Do(); err != nil {
+	if _, err := h.init.gitInfo.Do(context.Background()); err != nil {
 		return source.GitInfo{}, err
 	}
 
@@ -214,7 +214,7 @@ func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) {
 }
 
 func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) {
-	if _, err := h.init.gitInfo.Do(); err != nil {
+	if _, err := h.init.gitInfo.Do(context.Background()); err != nil {
 		return nil, err
 	}
 
@@ -363,15 +363,15 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		donec: make(chan bool),
 	}
 
-	h.init.data.Add(func() (any, error) {
+	h.init.data.Add(func(context.Context) (any, error) {
 		err := h.loadData(h.PathSpec.BaseFs.Data.Dirs)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load data: %w", err)
 		}
 		return nil, nil
 	})
 
-	h.init.layouts.Add(func() (any, error) {
+	h.init.layouts.Add(func(context.Context) (any, error) {
 		for _, s := range h.Sites {
 			if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil {
 				return nil, err
@@ -380,7 +380,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		return nil, nil
 	})
 
-	h.init.translations.Add(func() (any, error) {
+	h.init.translations.Add(func(context.Context) (any, error) {
 		if len(h.Sites) > 1 {
 			allTranslations := pagesToTranslationsMap(h.Sites)
 			assignTranslationsToPages(allTranslations, h.Sites)
@@ -389,7 +389,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 		return nil, nil
 	})
 
-	h.init.gitInfo.Add(func() (any, error) {
+	h.init.gitInfo.Add(func(context.Context) (any, error) {
 		err := h.loadGitInfo()
 		if err != nil {
 			return nil, fmt.Errorf("failed to load Git info: %w", err)

hugolib/hugo_sites_build.go

@@ -268,7 +268,7 @@ func (h *HugoSites) assemble(bcfg *BuildCfg) error {
 }
 
 func (h *HugoSites) render(config *BuildCfg) error {
-	if _, err := h.init.layouts.Do(); err != nil {
+	if _, err := h.init.layouts.Do(context.Background()); err != nil {
 		return err
 	}
 

hugolib/hugo_sites_build_errors_test.go

@@ -396,7 +396,7 @@ line 4
 
 }
 
-func TestErrorNestedShortocde(t *testing.T) {
+func TestErrorNestedShortcode(t *testing.T) {
 	t.Parallel()
 
 	files := `