From c02103f4baf48d3be7694f630d18452298ce08c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 11 Feb 2023 16:20:24 +0100 Subject: [PATCH] Add page fragments support to Related The main topic of this commit is that you can now index fragments (content heading identifiers) when calling `.Related`. You can do this by: * Configure one or more indices with type `fragments` * The name of those index configurations maps to an (optional) front matter slice with fragment references. This allows you to link page<->fragment and page<->page. * This also will index all the fragments (heading identifiers) of the pages. It's also possible to use type `fragments` indices in shortcode, e.g.: ``` {{ $related := site.RegularPages.Related .Page }} ``` But, and this is important, you need to include the shortcode using the `{{<` delimiter. Not doing so will create infinite loops and timeouts. This commit also: * Consolidates all `.Related*` methods into one, which takes either a `Page` or an options map as its only argument. * Add `context.Context` to all of the content related Page API. Turns out it wasn't strictly needed for this particular feature, but it will soon become usefil, e.g. in #9339. Closes #10711 Updates #9339 Updates #10725 --- common/collections/slice.go | 20 ++ common/collections/slice_test.go | 15 ++ go.mod | 1 + go.sum | 1 + hugolib/content_factory.go | 3 +- hugolib/content_map_page.go | 2 +- hugolib/embedded_shortcodes_test.go | 3 +- hugolib/hugo_sites.go | 14 +- hugolib/hugo_sites_build.go | 2 +- hugolib/hugo_sites_build_errors_test.go | 2 +- hugolib/image_test.go | 153 ----------- hugolib/language_content_dir_test.go | 3 +- hugolib/page.go | 55 +++- hugolib/page__content.go | 14 +- hugolib/page__menus.go | 7 +- hugolib/page__new.go | 3 +- hugolib/page__per_output.go | 219 ++++++++++------ hugolib/page__position.go | 6 +- hugolib/page_test.go | 53 ++-- hugolib/shortcode.go | 101 +++++--- hugolib/shortcode_page.go | 41 ++- hugolib/shortcode_test.go | 28 ++- hugolib/site.go | 22 +- hugolib/site_render.go | 3 +- hugolib/site_test.go | 7 +- hugolib/testhelpers_test.go | 3 +- lazy/init.go | 30 +-- lazy/init_test.go | 37 +-- markup/asciidocext/convert.go | 26 +- markup/asciidocext/convert_test.go | 82 +----- markup/converter/converter.go | 2 +- markup/goldmark/convert.go | 10 +- markup/goldmark/toc.go | 10 +- markup/tableofcontents/tableofcontents.go | 110 ++++++-- .../tableofcontents/tableofcontents_test.go | 85 ++++++- related/integration_test.go | 191 ++++++++++++++ related/inverted_index.go | 238 ++++++++++++------ related/inverted_index_test.go | 41 +-- resources/errorResource.go | 3 +- resources/image_test.go | 3 +- resources/page/page.go | 41 ++- resources/page/page_lazy_contentprovider.go | 81 +++--- resources/page/page_marshaljson.autogen.go | 34 --- resources/page/page_nop.go | 35 ++- resources/page/pagegroup.go | 10 +- resources/page/pagegroup_test.go | 15 +- resources/page/pages_related.go | 74 ++++-- resources/page/pages_related_test.go | 22 +- resources/page/pages_sort.go | 5 +- resources/page/pages_sort_test.go | 9 +- resources/page/pagination_test.go | 23 +- resources/page/testhelpers_test.go | 34 ++- resources/postpub/postpub.go | 5 +- resources/resource.go | 3 +- resources/resource/resourcetypes.go | 6 +- .../integrity/integrity_test.go | 3 +- .../minifier/minify_test.go | 3 +- resources/transform.go | 5 +- resources/transform_test.go | 21 +- .../texttemplate/hugo_template.go | 1 + tpl/transform/transform.go | 5 +- tpl/transform/transform_test.go | 5 +- 62 files changed, 1311 insertions(+), 778 deletions(-) create mode 100644 related/integration_test.go diff --git a/common/collections/slice.go b/common/collections/slice.go index 51cb6ec1f80..bf5c7b52b30 100644 --- a/common/collections/slice.go +++ b/common/collections/slice.go @@ -15,6 +15,7 @@ package collections import ( "reflect" + "sort" ) // Slicer defines a very generic way to create a typed slice. This is used @@ -74,3 +75,22 @@ func StringSliceToInterfaceSlice(ss []string) []any { return result } + +type SortedStringSlice []string + +// Contains returns true if s is in ss. +func (ss SortedStringSlice) Contains(s string) bool { + i := sort.SearchStrings(ss, s) + return i < len(ss) && ss[i] == s +} + +// Count returns the number of times s is in ss. +func (ss SortedStringSlice) Count(s string) int { + var count int + i := sort.SearchStrings(ss, s) + for i < len(ss) && ss[i] == s { + count++ + i++ + } + return count +} diff --git a/common/collections/slice_test.go b/common/collections/slice_test.go index 8e6553994da..5788b9161c8 100644 --- a/common/collections/slice_test.go +++ b/common/collections/slice_test.go @@ -122,3 +122,18 @@ func TestSlice(t *testing.T) { c.Assert(test.expected, qt.DeepEquals, result, errMsg) } } + +func TestSortedStringSlice(t *testing.T) { + t.Parallel() + c := qt.New(t) + + var s SortedStringSlice = []string{"a", "b", "b", "b", "c", "d"} + + c.Assert(s.Contains("a"), qt.IsTrue) + c.Assert(s.Contains("b"), qt.IsTrue) + c.Assert(s.Contains("z"), qt.IsFalse) + c.Assert(s.Count("b"), qt.Equals, 3) + c.Assert(s.Count("z"), qt.Equals, 0) + c.Assert(s.Count("a"), qt.Equals, 1) + +} diff --git a/go.mod b/go.mod index d53a7a06e65..5ae9efd269a 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( github.com/yuin/goldmark v1.5.4 go.uber.org/atomic v1.10.0 gocloud.dev v0.28.0 + golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 golang.org/x/image v0.0.0-20211028202545-6944b10bf410 golang.org/x/net v0.4.0 golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index 45d1f4ebeac..48c2ba125be 100644 --- a/go.sum +++ b/go.sum @@ -2002,6 +2002,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/hugolib/content_factory.go b/hugolib/content_factory.go index 017a0bc979c..e22f4651398 100644 --- a/hugolib/content_factory.go +++ b/hugolib/content_factory.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "io" "path/filepath" @@ -83,7 +84,7 @@ func (f ContentFactory) ApplyArchetypeTemplate(w io.Writer, p page.Page, archety return fmt.Errorf("failed to parse archetype template: %s: %w", err, err) } - result, err := executeToString(ps.s.Tmpl(), templ, d) + result, err := executeToString(context.TODO(), ps.s.Tmpl(), templ, d) if err != nil { return fmt.Errorf("failed to execute archetype template: %s: %w", err, err) } diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index d8f28286cdf..70c5d6a27a3 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -171,7 +171,7 @@ func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapB return nil, err } - ps.init.Add(func() (any, error) { + ps.init.Add(func(context.Context) (any, error) { pp, err := newPagePaths(s, ps, metaProvider) if err != nil { return nil, err diff --git a/hugolib/embedded_shortcodes_test.go b/hugolib/embedded_shortcodes_test.go index 1707bcfa7e8..1e06494bfa6 100644 --- a/hugolib/embedded_shortcodes_test.go +++ b/hugolib/embedded_shortcodes_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "encoding/json" "fmt" "html/template" @@ -70,7 +71,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) { c.Assert(len(s.RegularPages()), qt.Equals, 1) - content, err := s.RegularPages()[0].Content() + content, err := s.RegularPages()[0].Content(context.Background()) c.Assert(err, qt.IsNil) output := cast.ToString(content) diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 569c27be5c7..cdc5d97fb78 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -194,7 +194,7 @@ func (h *hugoSitesInit) Reset() { } func (h *HugoSites) Data() map[string]any { - if _, err := h.init.data.Do(); err != nil { + if _, err := h.init.data.Do(context.Background()); err != nil { h.SendError(fmt.Errorf("failed to load data: %w", err)) return nil } @@ -202,7 +202,7 @@ func (h *HugoSites) Data() map[string]any { } func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) { - if _, err := h.init.gitInfo.Do(); err != nil { + if _, err := h.init.gitInfo.Do(context.Background()); err != nil { return source.GitInfo{}, err } @@ -214,7 +214,7 @@ func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) { } func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) { - if _, err := h.init.gitInfo.Do(); err != nil { + if _, err := h.init.gitInfo.Do(context.Background()); err != nil { return nil, err } @@ -363,7 +363,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { donec: make(chan bool), } - h.init.data.Add(func() (any, error) { + h.init.data.Add(func(context.Context) (any, error) { err := h.loadData(h.PathSpec.BaseFs.Data.Dirs) if err != nil { return nil, fmt.Errorf("failed to load data: %w", err) @@ -371,7 +371,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { return nil, nil }) - h.init.layouts.Add(func() (any, error) { + h.init.layouts.Add(func(context.Context) (any, error) { for _, s := range h.Sites { if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil { return nil, err @@ -380,7 +380,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { return nil, nil }) - h.init.translations.Add(func() (any, error) { + h.init.translations.Add(func(context.Context) (any, error) { if len(h.Sites) > 1 { allTranslations := pagesToTranslationsMap(h.Sites) assignTranslationsToPages(allTranslations, h.Sites) @@ -389,7 +389,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { return nil, nil }) - h.init.gitInfo.Add(func() (any, error) { + h.init.gitInfo.Add(func(context.Context) (any, error) { err := h.loadGitInfo() if err != nil { return nil, fmt.Errorf("failed to load Git info: %w", err) diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 5eee564aaf3..66abf4f1655 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -268,7 +268,7 @@ func (h *HugoSites) assemble(bcfg *BuildCfg) error { } func (h *HugoSites) render(config *BuildCfg) error { - if _, err := h.init.layouts.Do(); err != nil { + if _, err := h.init.layouts.Do(context.Background()); err != nil { return err } diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index ffbfe1c17e5..f42b4446172 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -396,7 +396,7 @@ line 4 } -func TestErrorNestedShortocde(t *testing.T) { +func TestErrorNestedShortcode(t *testing.T) { t.Parallel() files := ` diff --git a/hugolib/image_test.go b/hugolib/image_test.go index ac18b9423d8..db1707c2292 100644 --- a/hugolib/image_test.go +++ b/hugolib/image_test.go @@ -14,162 +14,9 @@ package hugolib import ( - "io" - "os" - "path/filepath" - "runtime" - "strings" "testing" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/htesting" - - qt "github.com/frankban/quicktest" - "github.com/gohugoio/hugo/hugofs" ) -// We have many tests for the different resize operations etc. in the resource package, -// this is an integration test. -func TestImageOps(t *testing.T) { - c := qt.New(t) - // Make this a real as possible. - workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "image-resize") - c.Assert(err, qt.IsNil) - defer clean() - - newBuilder := func(timeout any) *sitesBuilder { - v := config.NewWithTestDefaults() - v.Set("workingDir", workDir) - v.Set("baseURL", "https://example.org") - v.Set("timeout", timeout) - - b := newTestSitesBuilder(t).WithWorkingDir(workDir) - b.Fs = hugofs.NewDefault(v) - b.WithViper(v) - b.WithContent("mybundle/index.md", ` ---- -title: "My bundle" ---- - -{{< imgproc >}} - -`) - - b.WithTemplatesAdded( - "shortcodes/imgproc.html", ` -{{ $img := resources.Get "images/sunset.jpg" }} -{{ $r := $img.Resize "129x239" }} -IMG SHORTCODE: {{ $r.RelPermalink }}/{{ $r.Width }} -`, - "index.html", ` -{{ $p := .Site.GetPage "mybundle" }} -{{ $img1 := resources.Get "images/sunset.jpg" }} -{{ $img2 := $p.Resources.GetMatch "sunset.jpg" }} -{{ $img3 := resources.GetMatch "images/*.jpg" }} -{{ $r := $img1.Resize "123x234" }} -{{ $r2 := $r.Resize "12x23" }} -{{ $b := $img2.Resize "345x678" }} -{{ $b2 := $b.Resize "34x67" }} -{{ $c := $img3.Resize "456x789" }} -{{ $fingerprinted := $img1.Resize "350x" | fingerprint }} - -{{ $images := slice $r $r2 $b $b2 $c $fingerprinted }} - -{{ range $i, $r := $images }} -{{ printf "Resized%d:" (add $i 1) }} {{ $r.Name }}|{{ $r.Width }}|{{ $r.Height }}|{{ $r.MediaType }}|{{ $r.RelPermalink }}| -{{ end }} - -{{ $blurryGrayscale1 := $r | images.Filter images.Grayscale (images.GaussianBlur 8) }} -BG1: {{ $blurryGrayscale1.RelPermalink }}/{{ $blurryGrayscale1.Width }} -{{ $blurryGrayscale2 := $r.Filter images.Grayscale (images.GaussianBlur 8) }} -BG2: {{ $blurryGrayscale2.RelPermalink }}/{{ $blurryGrayscale2.Width }} -{{ $blurryGrayscale2_2 := $r.Filter images.Grayscale (images.GaussianBlur 8) }} -BG2_2: {{ $blurryGrayscale2_2.RelPermalink }}/{{ $blurryGrayscale2_2.Width }} - -{{ $filters := slice images.Grayscale (images.GaussianBlur 9) }} -{{ $blurryGrayscale3 := $r | images.Filter $filters }} -BG3: {{ $blurryGrayscale3.RelPermalink }}/{{ $blurryGrayscale3.Width }} - -{{ $blurryGrayscale4 := $r.Filter $filters }} -BG4: {{ $blurryGrayscale4.RelPermalink }}/{{ $blurryGrayscale4.Width }} - -{{ $p.Content }} - -`) - - return b - } - - imageDir := filepath.Join(workDir, "assets", "images") - bundleDir := filepath.Join(workDir, "content", "mybundle") - - c.Assert(os.MkdirAll(imageDir, 0777), qt.IsNil) - c.Assert(os.MkdirAll(bundleDir, 0777), qt.IsNil) - src, err := os.Open("testdata/sunset.jpg") - c.Assert(err, qt.IsNil) - out, err := os.Create(filepath.Join(imageDir, "sunset.jpg")) - c.Assert(err, qt.IsNil) - _, err = io.Copy(out, src) - c.Assert(err, qt.IsNil) - out.Close() - - src.Seek(0, 0) - - out, err = os.Create(filepath.Join(bundleDir, "sunset.jpg")) - c.Assert(err, qt.IsNil) - _, err = io.Copy(out, src) - c.Assert(err, qt.IsNil) - out.Close() - src.Close() - - // First build it with a very short timeout to trigger errors. - b := newBuilder("10ns") - - imgExpect := ` -Resized1: images/sunset.jpg|123|234|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_123x234_resize_q75_box.jpg| -Resized2: images/sunset.jpg|12|23|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ada4bb1a57f77a63306e3bd67286248e.jpg| -Resized3: sunset.jpg|345|678|image/jpeg|/mybundle/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_345x678_resize_q75_box.jpg| -Resized4: sunset.jpg|34|67|image/jpeg|/mybundle/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_44d8c928664d7c5a67377c6ec58425ce.jpg| -Resized5: images/sunset.jpg|456|789|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_456x789_resize_q75_box.jpg| -Resized6: images/sunset.jpg|350|219|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_350x0_resize_q75_box.a86fe88d894e5db613f6aa8a80538fefc25b20fa24ba0d782c057adcef616f56.jpg| -BG1: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_2ae8bb993431ec1aec40fe59927b46b4.jpg/123 -BG2: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_2ae8bb993431ec1aec40fe59927b46b4.jpg/123 -BG3: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg/123 -BG4: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg/123 -IMG SHORTCODE: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_129x239_resize_q75_box.jpg/129 -` - - assertImages := func() { - b.Helper() - b.AssertFileContent("public/index.html", imgExpect) - b.AssertImage(350, 219, "public/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_350x0_resize_q75_box.a86fe88d894e5db613f6aa8a80538fefc25b20fa24ba0d782c057adcef616f56.jpg") - b.AssertImage(129, 239, "public/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_129x239_resize_q75_box.jpg") - } - - err = b.BuildE(BuildCfg{}) - if runtime.GOOS != "windows" && !strings.Contains(runtime.GOARCH, "arm") && !htesting.IsGitHubAction() { - // TODO(bep) - c.Assert(err, qt.Not(qt.IsNil)) - } - - b = newBuilder(29000) - b.Build(BuildCfg{}) - - assertImages() - - // Truncate one image. - imgInCache := filepath.Join(workDir, "resources/_gen/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg") - f, err := os.Create(imgInCache) - c.Assert(err, qt.IsNil) - f.Close() - - // Build it again to make sure we read images from file cache. - b = newBuilder("30s") - b.Build(BuildCfg{}) - - assertImages() -} - func TestImageResizeMultilingual(t *testing.T) { b := newTestSitesBuilder(t).WithConfigFile("toml", ` baseURL="https://example.org" diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go index 57cdab67bc6..23809f4dffe 100644 --- a/hugolib/language_content_dir_test.go +++ b/hugolib/language_content_dir_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "os" "path/filepath" @@ -245,7 +246,7 @@ Content. c.Assert(svP2.Language().Lang, qt.Equals, "sv") c.Assert(nnP2.Language().Lang, qt.Equals, "nn") - content, _ := nnP2.Content() + content, _ := nnP2.Content(context.Background()) contentStr := cast.ToString(content) c.Assert(contentStr, qt.Contains, "SVP3-REF: https://example.org/sv/sect/p-sv-3/") c.Assert(contentStr, qt.Contains, "SVP3-RELREF: /sv/sect/p-sv-3/") diff --git a/hugolib/page.go b/hugolib/page.go index 97f1ed351ba..35535d5d681 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -15,6 +15,7 @@ package hugolib import ( "bytes" + "context" "fmt" "path" "path/filepath" @@ -24,8 +25,10 @@ import ( "go.uber.org/atomic" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/related" "github.com/gohugoio/hugo/markup/converter" + "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/gohugoio/hugo/tpl" @@ -148,6 +151,50 @@ func (p *pageState) GetIdentity() identity.Identity { return identity.NewPathIdentity(files.ComponentFolderContent, filepath.FromSlash(p.Pathc())) } +func (p *pageState) Fragments(ctx context.Context) *tableofcontents.Fragments { + p.s.initInit(ctx, p.cp.initToC, p) + if p.pageOutput.cp.tableOfContents == nil { + return tableofcontents.Empty + } + return p.pageOutput.cp.tableOfContents +} + +func (p *pageState) HeadingsFiltered(context.Context) tableofcontents.Headings { + return nil +} + +type pageFragment struct { + fragment string + relPermalink string + permalink string + title string +} + +type pageFragmentProvider struct { + *pageState + headings tableofcontents.Headings +} + +func (p *pageFragmentProvider) HeadingsFiltered(context.Context) tableofcontents.Headings { + return p.headings +} + +func (p *pageFragmentProvider) page() page.Page { + return p.pageState +} + +// For internal use by the related content feature. +func (p *pageState) ApplyFilterToHeadings(ctx context.Context, fn func(*tableofcontents.Heading) bool) related.Document { + if p.pageOutput.cp.tableOfContents == nil { + return p + } + headings := p.pageOutput.cp.tableOfContents.Headings.FilterBy(fn) + return &pageFragmentProvider{ + pageState: p, + headings: headings, + } +} + func (p *pageState) GitInfo() source.GitInfo { return p.gitInfo } @@ -351,7 +398,7 @@ func (p *pageState) String() string { // IsTranslated returns whether this content file is translated to // other language(s). func (p *pageState) IsTranslated() bool { - p.s.h.init.translations.Do() + p.s.h.init.translations.Do(context.Background()) return len(p.translations) > 0 } @@ -375,13 +422,13 @@ func (p *pageState) TranslationKey() string { // AllTranslations returns all translations, including the current Page. func (p *pageState) AllTranslations() page.Pages { - p.s.h.init.translations.Do() + p.s.h.init.translations.Do(context.Background()) return p.allTranslations } // Translations returns the translations excluding the current Page. func (p *pageState) Translations() page.Pages { - p.s.h.init.translations.Do() + p.s.h.init.translations.Do(context.Background()) return p.translations } @@ -461,7 +508,7 @@ func (p *pageState) initOutputFormat(isRenderingSite bool, idx int) error { // Must be run after the site section tree etc. is built and ready. func (p *pageState) initPage() error { - if _, err := p.init.Do(); err != nil { + if _, err := p.init.Do(context.Background()); err != nil { return err } return nil diff --git a/hugolib/page__content.go b/hugolib/page__content.go index a721d1fce97..89c38bd841b 100644 --- a/hugolib/page__content.go +++ b/hugolib/page__content.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "github.com/gohugoio/hugo/output" @@ -37,9 +38,9 @@ type pageContent struct { } // returns the content to be processed by Goldmark or similar. -func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMap, renderedShortcodes map[string]string) []byte { +func (p pageContent) contentToRender(ctx context.Context, parsed pageparser.Result, pm *pageContentMap, renderedShortcodes map[string]shortcodeRenderer) ([]byte, bool, error) { source := parsed.Input() - + var hasVariants bool c := make([]byte, 0, len(source)+(len(source)/10)) for _, it := range pm.items { @@ -57,7 +58,12 @@ func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMa panic(fmt.Sprintf("rendered shortcode %q not found", v.placeholder)) } - c = append(c, []byte(renderedShortcode)...) + b, more, err := renderedShortcode.renderShortcode(ctx) + if err != nil { + return nil, false, fmt.Errorf("failed to render shortcode: %w", err) + } + hasVariants = hasVariants || more + c = append(c, []byte(b)...) } else { // Insert the placeholder so we can insert the content after @@ -69,7 +75,7 @@ func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMa } } - return c + return c, hasVariants, nil } func (p pageContent) selfLayoutForOutput(f output.Format) string { diff --git a/hugolib/page__menus.go b/hugolib/page__menus.go index 49d392c2fc2..5bed2bc033d 100644 --- a/hugolib/page__menus.go +++ b/hugolib/page__menus.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "sync" "github.com/gohugoio/hugo/navigation" @@ -29,13 +30,13 @@ type pageMenus struct { } func (p *pageMenus) HasMenuCurrent(menuID string, me *navigation.MenuEntry) bool { - p.p.s.init.menus.Do() + p.p.s.init.menus.Do(context.Background()) p.init() return p.q.HasMenuCurrent(menuID, me) } func (p *pageMenus) IsMenuCurrent(menuID string, inme *navigation.MenuEntry) bool { - p.p.s.init.menus.Do() + p.p.s.init.menus.Do(context.Background()) p.init() return p.q.IsMenuCurrent(menuID, inme) } @@ -43,7 +44,7 @@ func (p *pageMenus) IsMenuCurrent(menuID string, inme *navigation.MenuEntry) boo func (p *pageMenus) Menus() navigation.PageMenus { // There is a reverse dependency here. initMenus will, once, build the // site menus and update any relevant page. - p.p.s.init.menus.Do() + p.p.s.init.menus.Do(context.Background()) return p.menus() } diff --git a/hugolib/page__new.go b/hugolib/page__new.go index e52b9476b03..3787cd2bd4f 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "html/template" "strings" @@ -121,7 +122,7 @@ func newPageFromMeta( return nil, err } - ps.init.Add(func() (any, error) { + ps.init.Add(func(context.Context) (any, error) { pp, err := newPagePaths(metaProvider.s, ps, metaProvider) if err != nil { return nil, err diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index 97e9cc46564..827a6b792fb 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "html/template" - "runtime/debug" "strings" "sync" "unicode/utf8" @@ -34,6 +33,7 @@ import ( "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/markup/highlight/chromalexers" + "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/gohugoio/hugo/markup/converter" @@ -87,43 +87,35 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err renderHooks: &renderHooks{}, } - initContent := func() (err error) { - p.s.h.IncrContentRender() - + initToC := func(ctx context.Context) (err error) { if p.cmap == nil { // Nothing to do. return nil } - defer func() { - // See https://github.com/gohugoio/hugo/issues/6210 - if r := recover(); r != nil { - err = fmt.Errorf("%s", r) - p.s.Log.Errorf("[BUG] Got panic:\n%s\n%s", r, string(debug.Stack())) - } - }() if err := po.cp.initRenderHooks(); err != nil { return err } - var hasShortcodeVariants bool - f := po.f - cp.contentPlaceholders, hasShortcodeVariants, err = p.shortcodeState.renderShortcodesForPage(p, f) + cp.contentPlaceholders, err = p.shortcodeState.prepareShortcodesForPage(ctx, p, f) if err != nil { return err } - if hasShortcodeVariants { + var hasVariants bool + cp.workContent, hasVariants, err = p.contentToRender(ctx, p.source.parsed, p.cmap, cp.contentPlaceholders) + if err != nil { + return err + } + if hasVariants { p.pageOutputTemplateVariationsState.Store(2) } - cp.workContent = p.contentToRender(p.source.parsed, p.cmap, cp.contentPlaceholders) - isHTML := cp.p.m.markup == "html" if !isHTML { - r, err := po.contentRenderer.RenderContent(cp.workContent, true) + r, err := po.contentRenderer.RenderContent(ctx, cp.workContent, true) if err != nil { return err } @@ -132,8 +124,9 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err if tocProvider, ok := r.(converter.TableOfContentsProvider); ok { cfg := p.s.ContentSpec.Converters.GetMarkupConfig() - cp.tableOfContents = template.HTML( - tocProvider.TableOfContents().ToHTML( + cp.tableOfContents = tocProvider.TableOfContents() + cp.tableOfContentsHTML = template.HTML( + cp.tableOfContents.ToHTML( cfg.TableOfContents.StartLevel, cfg.TableOfContents.EndLevel, cfg.TableOfContents.Ordered, @@ -141,26 +134,60 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err ) } else { tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent) - cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents) + cp.tableOfContentsHTML = helpers.BytesToHTML(tmpTableOfContents) + cp.tableOfContents = tableofcontents.Empty cp.workContent = tmpContent } } - if cp.placeholdersEnabled { - // ToC was accessed via .Page.TableOfContents in the shortcode, - // at a time when the ToC wasn't ready. - cp.contentPlaceholders[tocShortcodePlaceholder] = string(cp.tableOfContents) + return nil + + } + + initContent := func(ctx context.Context) (err error) { + + p.s.h.IncrContentRender() + + if p.cmap == nil { + // Nothing to do. + return nil } if p.cmap.hasNonMarkdownShortcode || cp.placeholdersEnabled { // There are one or more replacement tokens to be replaced. - cp.workContent, err = replaceShortcodeTokens(cp.workContent, cp.contentPlaceholders) + var hasShortcodeVariants bool + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + if token == tocShortcodePlaceholder { + // The Page's TableOfContents was accessed in a shortcode. + if cp.tableOfContentsHTML == "" { + cp.p.s.initInit(ctx, cp.initToC, cp.p) + } + return []byte(cp.tableOfContentsHTML), nil + } + renderer, found := cp.contentPlaceholders[token] + if found { + repl, more, err := renderer.renderShortcode(ctx) + if err != nil { + return nil, err + } + hasShortcodeVariants = hasShortcodeVariants || more + return repl, nil + } + // This should never happen. + return nil, fmt.Errorf("unknown shortcode token %q", token) + } + + cp.workContent, err = expandShortcodeTokens(ctx, cp.workContent, tokenHandler) if err != nil { return err } + if hasShortcodeVariants { + p.pageOutputTemplateVariationsState.Store(2) + } } if cp.p.source.hasSummaryDivider { + isHTML := cp.p.m.markup == "html" if isHTML { src := p.source.parsed.Input() @@ -183,7 +210,7 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err } } } else if cp.p.m.summary != "" { - b, err := po.contentRenderer.RenderContent([]byte(cp.p.m.summary), false) + b, err := po.contentRenderer.RenderContent(ctx, []byte(cp.p.m.summary), false) if err != nil { return err } @@ -196,12 +223,16 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err return nil } + cp.initToC = parent.Branch(func(ctx context.Context) (any, error) { + return nil, initToC(ctx) + }) + // There may be recursive loops in shortcodes and render hooks. - cp.initMain = parent.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (any, error) { - return nil, initContent() + cp.initMain = cp.initToC.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (any, error) { + return nil, initContent(ctx) }) - cp.initPlain = cp.initMain.Branch(func() (any, error) { + cp.initPlain = cp.initMain.Branch(func(context.Context) (any, error) { cp.plain = tpl.StripHTML(string(cp.content)) cp.plainWords = strings.Fields(cp.plain) cp.setWordCounts(p.m.isCJKLanguage) @@ -228,6 +259,7 @@ type pageContentOutput struct { p *pageState // Lazy load dependencies + initToC *lazy.Init initMain *lazy.Init initPlain *lazy.Init @@ -243,12 +275,13 @@ type pageContentOutput struct { // Temporary storage of placeholders mapped to their content. // These are shortcodes etc. Some of these will need to be replaced // after any markup is rendered, so they share a common prefix. - contentPlaceholders map[string]string + contentPlaceholders map[string]shortcodeRenderer // Content sections - content template.HTML - summary template.HTML - tableOfContents template.HTML + content template.HTML + summary template.HTML + tableOfContents *tableofcontents.Fragments + tableOfContentsHTML template.HTML truncated bool @@ -263,76 +296,76 @@ func (p *pageContentOutput) trackDependency(id identity.Provider) { if p.dependencyTracker != nil { p.dependencyTracker.Add(id) } + } func (p *pageContentOutput) Reset() { if p.dependencyTracker != nil { p.dependencyTracker.Reset() } + p.initToC.Reset() p.initMain.Reset() p.initPlain.Reset() p.renderHooks = &renderHooks{} } -func (p *pageContentOutput) Content() (any, error) { - if p.p.s.initInit(p.initMain, p.p) { - return p.content, nil - } - return nil, nil +func (p *pageContentOutput) Content(ctx context.Context) (any, error) { + p.p.s.initInit(ctx, p.initMain, p.p) + return p.content, nil } -func (p *pageContentOutput) FuzzyWordCount() int { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) FuzzyWordCount(ctx context.Context) int { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.fuzzyWordCount } -func (p *pageContentOutput) Len() int { - p.p.s.initInit(p.initMain, p.p) +func (p *pageContentOutput) Len(ctx context.Context) int { + p.p.s.initInit(ctx, p.initMain, p.p) return len(p.content) } -func (p *pageContentOutput) Plain() string { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) Plain(ctx context.Context) string { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.plain } -func (p *pageContentOutput) PlainWords() []string { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) PlainWords(ctx context.Context) []string { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.plainWords } -func (p *pageContentOutput) ReadingTime() int { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) ReadingTime(ctx context.Context) int { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.readingTime } -func (p *pageContentOutput) Summary() template.HTML { - p.p.s.initInit(p.initMain, p.p) +func (p *pageContentOutput) Summary(ctx context.Context) template.HTML { + p.p.s.initInit(ctx, p.initMain, p.p) if !p.p.source.hasSummaryDivider { - p.p.s.initInit(p.initPlain, p.p) + p.p.s.initInit(ctx, p.initPlain, p.p) } return p.summary } -func (p *pageContentOutput) TableOfContents() template.HTML { - p.p.s.initInit(p.initMain, p.p) - return p.tableOfContents +func (p *pageContentOutput) TableOfContents(ctx context.Context) template.HTML { + p.p.s.initInit(ctx, p.initMain, p.p) + return p.tableOfContentsHTML } -func (p *pageContentOutput) Truncated() bool { +func (p *pageContentOutput) Truncated(ctx context.Context) bool { if p.p.truncated { return true } - p.p.s.initInit(p.initPlain, p.p) + p.p.s.initInit(ctx, p.initPlain, p.p) return p.truncated } -func (p *pageContentOutput) WordCount() int { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) WordCount(ctx context.Context) int { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.wordCount } -func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) { +func (p *pageContentOutput) RenderString(ctx context.Context, args ...any) (template.HTML, error) { if len(args) < 1 || len(args) > 2 { return "", errors.New("want 1 or 2 arguments") } @@ -405,42 +438,62 @@ func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) { return "", err } - placeholders, hasShortcodeVariants, err := s.renderShortcodesForPage(p.p, p.f) + placeholders, err := s.prepareShortcodesForPage(ctx, p.p, p.f) if err != nil { return "", err } - if hasShortcodeVariants { + contentToRender, hasVariants, err := p.p.contentToRender(ctx, parsed, pm, placeholders) + if err != nil { + return "", err + } + if hasVariants { p.p.pageOutputTemplateVariationsState.Store(2) } - - b, err := p.renderContentWithConverter(conv, p.p.contentToRender(parsed, pm, placeholders), false) + b, err := p.renderContentWithConverter(ctx, conv, contentToRender, false) if err != nil { return "", p.p.wrapError(err) } rendered = b.Bytes() - if p.placeholdersEnabled { - // ToC was accessed via .Page.TableOfContents in the shortcode, - // at a time when the ToC wasn't ready. - if _, err := p.p.Content(); err != nil { - return "", err + if pm.hasNonMarkdownShortcode || p.placeholdersEnabled { + var hasShortcodeVariants bool + + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + if token == tocShortcodePlaceholder { + // The Page's TableOfContents was accessed in a shortcode. + if p.tableOfContentsHTML == "" { + p.p.s.initInit(ctx, p.initToC, p.p) + } + return []byte(p.tableOfContentsHTML), nil + } + renderer, found := placeholders[token] + if found { + repl, more, err := renderer.renderShortcode(ctx) + if err != nil { + return nil, err + } + hasShortcodeVariants = hasShortcodeVariants || more + return repl, nil + } + // This should not happen. + return nil, fmt.Errorf("unknown shortcode token %q", token) } - placeholders[tocShortcodePlaceholder] = string(p.tableOfContents) - } - if pm.hasNonMarkdownShortcode || p.placeholdersEnabled { - rendered, err = replaceShortcodeTokens(rendered, placeholders) + rendered, err = expandShortcodeTokens(ctx, rendered, tokenHandler) if err != nil { return "", err } + if hasShortcodeVariants { + p.p.pageOutputTemplateVariationsState.Store(2) + } } // We need a consolidated view in $page.HasShortcode p.p.shortcodeState.transferNames(s) } else { - c, err := p.renderContentWithConverter(conv, []byte(contentToRender), false) + c, err := p.renderContentWithConverter(ctx, conv, []byte(contentToRender), false) if err != nil { return "", p.p.wrapError(err) } @@ -457,12 +510,12 @@ func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) { return template.HTML(string(rendered)), nil } -func (p *pageContentOutput) RenderWithTemplateInfo(info tpl.Info, layout ...string) (template.HTML, error) { +func (p *pageContentOutput) RenderWithTemplateInfo(ctx context.Context, info tpl.Info, layout ...string) (template.HTML, error) { p.p.addDependency(info) - return p.Render(layout...) + return p.Render(ctx, layout...) } -func (p *pageContentOutput) Render(layout ...string) (template.HTML, error) { +func (p *pageContentOutput) Render(ctx context.Context, layout ...string) (template.HTML, error) { templ, found, err := p.p.resolveTemplate(layout...) if err != nil { return "", p.p.wrapError(err) @@ -475,7 +528,7 @@ func (p *pageContentOutput) Render(layout ...string) (template.HTML, error) { p.p.addDependency(templ.(tpl.Info)) // Make sure to send the *pageState and not the *pageContentOutput to the template. - res, err := executeToString(p.p.s.Tmpl(), templ, p.p) + res, err := executeToString(ctx, p.p.s.Tmpl(), templ, p.p) if err != nil { return "", p.p.wrapError(fmt.Errorf("failed to execute template %s: %w", templ.Name(), err)) } @@ -629,15 +682,15 @@ func (p *pageContentOutput) setAutoSummary() error { return nil } -func (cp *pageContentOutput) RenderContent(content []byte, renderTOC bool) (converter.Result, error) { +func (cp *pageContentOutput) RenderContent(ctx context.Context, content []byte, renderTOC bool) (converter.Result, error) { if err := cp.initRenderHooks(); err != nil { return nil, err } c := cp.p.getContentConverter() - return cp.renderContentWithConverter(c, content, renderTOC) + return cp.renderContentWithConverter(ctx, c, content, renderTOC) } -func (cp *pageContentOutput) renderContentWithConverter(c converter.Converter, content []byte, renderTOC bool) (converter.Result, error) { +func (cp *pageContentOutput) renderContentWithConverter(ctx context.Context, c converter.Converter, content []byte, renderTOC bool) (converter.Result, error) { r, err := c.Convert( converter.RenderContext{ Src: content, @@ -711,10 +764,10 @@ func (t targetPathsHolder) targetPaths() page.TargetPaths { return t.paths } -func executeToString(h tpl.TemplateHandler, templ tpl.Template, data any) (string, error) { +func executeToString(ctx context.Context, h tpl.TemplateHandler, templ tpl.Template, data any) (string, error) { b := bp.GetBuffer() defer bp.PutBuffer(b) - if err := h.Execute(templ, b, data); err != nil { + if err := h.ExecuteWithContext(ctx, templ, b, data); err != nil { return "", err } return b.String(), nil diff --git a/hugolib/page__position.go b/hugolib/page__position.go index a087872cc0a..d977a705273 100644 --- a/hugolib/page__position.go +++ b/hugolib/page__position.go @@ -14,6 +14,8 @@ package hugolib import ( + "context" + "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/resources/page" ) @@ -33,12 +35,12 @@ type nextPrev struct { } func (n *nextPrev) next() page.Page { - n.init.Do() + n.init.Do(context.Background()) return n.nextPage } func (n *nextPrev) prev() page.Page { - n.init.Do() + n.init.Do(context.Background()) return n.prevPage } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 939d06d4141..49617f17e21 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "html/template" "os" @@ -311,13 +312,13 @@ func normalizeContent(c string) string { func checkPageTOC(t *testing.T, page page.Page, toc string) { t.Helper() - if page.TableOfContents() != template.HTML(toc) { - t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(), toc) + if page.TableOfContents(context.Background()) != template.HTML(toc) { + t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(context.Background()), toc) } } func checkPageSummary(t *testing.T, page page.Page, summary string, msg ...any) { - a := normalizeContent(string(page.Summary())) + a := normalizeContent(string(page.Summary(context.Background()))) b := normalizeContent(summary) if a != b { t.Fatalf("Page summary is:\n%q.\nExpected\n%q (%q)", a, b, msg) @@ -443,9 +444,9 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { p := s.RegularPages()[0] - if p.Summary() != template.HTML( + if p.Summary(context.Background()) != template.HTML( "

The best static site generator.1

") { - t.Fatalf("Got summary:\n%q", p.Summary()) + t.Fatalf("Got summary:\n%q", p.Summary(context.Background())) } cnt := content(p) @@ -719,7 +720,7 @@ func TestSummaryWithHTMLTagsOnNextLine(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { c := qt.New(t) p := pages[0] - s := string(p.Summary()) + s := string(p.Summary(context.Background())) c.Assert(s, qt.Contains, "Happy new year everyone!") c.Assert(s, qt.Not(qt.Contains), "User interface") } @@ -1122,8 +1123,8 @@ func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { t.Parallel() assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 8 { - t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 8, p.WordCount()) + if p.WordCount(context.Background()) != 8 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 8, p.WordCount(context.Background())) } } @@ -1136,8 +1137,8 @@ func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 15 { - t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 15, p.WordCount()) + if p.WordCount(context.Background()) != 15 { + t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 15, p.WordCount(context.Background())) } } testAllMarkdownEnginesForPages(t, assertFunc, settings, simplePageWithAllCJKRunes) @@ -1149,13 +1150,13 @@ func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 74 { - t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 74, p.WordCount()) + if p.WordCount(context.Background()) != 74 { + t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 74, p.WordCount(context.Background())) } - if p.Summary() != simplePageWithMainEnglishWithCJKRunesSummary { - t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(), - simplePageWithMainEnglishWithCJKRunesSummary, p.Summary()) + if p.Summary(context.Background()) != simplePageWithMainEnglishWithCJKRunesSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(context.Background()), + simplePageWithMainEnglishWithCJKRunesSummary, p.Summary(context.Background())) } } @@ -1170,13 +1171,13 @@ func TestWordCountWithIsCJKLanguageFalse(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 75 { - t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.Plain(), 74, p.WordCount()) + if p.WordCount(context.Background()) != 75 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.Plain(context.Background()), 74, p.WordCount(context.Background())) } - if p.Summary() != simplePageWithIsCJKLanguageFalseSummary { - t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(), - simplePageWithIsCJKLanguageFalseSummary, p.Summary()) + if p.Summary(context.Background()) != simplePageWithIsCJKLanguageFalseSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(context.Background()), + simplePageWithIsCJKLanguageFalseSummary, p.Summary(context.Background())) } } @@ -1187,16 +1188,16 @@ func TestWordCount(t *testing.T) { t.Parallel() assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 483 { - t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount()) + if p.WordCount(context.Background()) != 483 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount(context.Background())) } - if p.FuzzyWordCount() != 500 { - t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 500, p.FuzzyWordCount()) + if p.FuzzyWordCount(context.Background()) != 500 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 500, p.FuzzyWordCount(context.Background())) } - if p.ReadingTime() != 3 { - t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime()) + if p.ReadingTime(context.Background()) != 3 { + t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime(context.Background())) } } diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 2951a143608..a82caff43cf 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -15,6 +15,7 @@ package hugolib import ( "bytes" + "context" "fmt" "html/template" "path" @@ -302,13 +303,44 @@ const ( innerCleanupExpand = "$1" ) -func renderShortcode( +func prepareShortcode( + ctx context.Context, level int, s *Site, tplVariants tpl.TemplateVariants, sc *shortcode, parent *ShortcodeWithPage, - p *pageState) (string, bool, error) { + p *pageState) (shortcodeRenderer, error) { + + toParseErr := func(err error) error { + return p.parseError(fmt.Errorf("failed to render shortcode %q: %w", sc.name, err), p.source.parsed.Input(), sc.pos) + } + + // Allow the caller to delay the rendering of the shortcode if needed. + var fn shortcodeRenderFunc = func(ctx context.Context) ([]byte, bool, error) { + r, err := doRenderShortcode(ctx, level, s, tplVariants, sc, parent, p) + if err != nil { + return nil, false, toParseErr(err) + } + b, hasVariants, err := r.renderShortcode(ctx) + if err != nil { + return nil, false, toParseErr(err) + } + return b, hasVariants, nil + } + + return fn, nil + +} + +func doRenderShortcode( + ctx context.Context, + level int, + s *Site, + tplVariants tpl.TemplateVariants, + sc *shortcode, + parent *ShortcodeWithPage, + p *pageState) (shortcodeRenderer, error) { var tmpl tpl.Template // Tracks whether this shortcode or any of its children has template variations @@ -319,7 +351,7 @@ func renderShortcode( if sc.isInline { if !p.s.ExecHelper.Sec().EnableInlineShortcodes { - return "", false, nil + return zeroShortcode, nil } templName := path.Join("_inline_shortcode", p.File().Path(), sc.name) if sc.isClosing { @@ -332,7 +364,7 @@ func renderShortcode( pos := fe.Position() pos.LineNumber += p.posOffset(sc.pos).LineNumber fe = fe.UpdatePosition(pos) - return "", false, p.wrapError(fe) + return zeroShortcode, p.wrapError(fe) } } else { @@ -340,7 +372,7 @@ func renderShortcode( var found bool tmpl, found = s.TextTmpl().Lookup(templName) if !found { - return "", false, fmt.Errorf("no earlier definition of shortcode %q found", sc.name) + return zeroShortcode, fmt.Errorf("no earlier definition of shortcode %q found", sc.name) } } } else { @@ -348,7 +380,7 @@ func renderShortcode( tmpl, found, more = s.Tmpl().LookupVariant(sc.name, tplVariants) if !found { s.Log.Errorf("Unable to locate template for shortcode %q in page %q", sc.name, p.File().Path()) - return "", false, nil + return zeroShortcode, nil } hasVariants = hasVariants || more } @@ -365,16 +397,20 @@ func renderShortcode( case string: inner += innerData case *shortcode: - s, more, err := renderShortcode(level+1, s, tplVariants, innerData, data, p) + s, err := prepareShortcode(ctx, level+1, s, tplVariants, innerData, data, p) if err != nil { - return "", false, err + return zeroShortcode, err } + ss, more, err := s.renderShortcodeString(ctx) hasVariants = hasVariants || more - inner += s + if err != nil { + return zeroShortcode, err + } + inner += ss default: s.Log.Errorf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", sc.name, p.File().Path(), reflect.TypeOf(innerData)) - return "", false, nil + return zeroShortcode, nil } } @@ -382,9 +418,9 @@ func renderShortcode( // shortcode. if sc.doMarkup && (level > 0 || sc.configVersion() == 1) { var err error - b, err := p.pageOutput.contentRenderer.RenderContent([]byte(inner), false) + b, err := p.pageOutput.contentRenderer.RenderContent(ctx, []byte(inner), false) if err != nil { - return "", false, err + return zeroShortcode, err } newInner := b.Bytes() @@ -418,14 +454,14 @@ func renderShortcode( } - result, err := renderShortcodeWithPage(s.Tmpl(), tmpl, data) + result, err := renderShortcodeWithPage(ctx, s.Tmpl(), tmpl, data) if err != nil && sc.isInline { fe := herrors.NewFileErrorFromName(err, p.File().Filename()) pos := fe.Position() pos.LineNumber += p.posOffset(sc.pos).LineNumber fe = fe.UpdatePosition(pos) - return "", false, fe + return zeroShortcode, fe } if len(sc.inner) == 0 && len(sc.indentation) > 0 { @@ -444,7 +480,7 @@ func renderShortcode( bp.PutBuffer(b) } - return result, hasVariants, err + return prerenderedShortcode{s: result, hasVariants: hasVariants}, err } func (s *shortcodeHandler) hasShortcodes() bool { @@ -473,28 +509,24 @@ func (s *shortcodeHandler) hasName(name string) bool { return ok } -func (s *shortcodeHandler) renderShortcodesForPage(p *pageState, f output.Format) (map[string]string, bool, error) { - rendered := make(map[string]string) +func (s *shortcodeHandler) prepareShortcodesForPage(ctx context.Context, p *pageState, f output.Format) (map[string]shortcodeRenderer, error) { + rendered := make(map[string]shortcodeRenderer) tplVariants := tpl.TemplateVariants{ Language: p.Language().Lang, OutputFormat: f, } - var hasVariants bool - for _, v := range s.shortcodes { - s, more, err := renderShortcode(0, s.s, tplVariants, v, nil, p) + s, err := prepareShortcode(ctx, 0, s.s, tplVariants, v, nil, p) if err != nil { - err = p.parseError(fmt.Errorf("failed to render shortcode %q: %w", v.name, err), p.source.parsed.Input(), v.pos) - return nil, false, err + return nil, err } - hasVariants = hasVariants || more rendered[v.placeholder] = s } - return rendered, hasVariants, nil + return rendered, nil } func (s *shortcodeHandler) parseError(err error, input []byte, pos int) error { @@ -668,11 +700,11 @@ Loop: // Replace prefixed shortcode tokens with the real content. // Note: This function will rewrite the input slice. -func replaceShortcodeTokens(source []byte, replacements map[string]string) ([]byte, error) { - if len(replacements) == 0 { - return source, nil - } - +func expandShortcodeTokens( + ctx context.Context, + source []byte, + tokenHandler func(ctx context.Context, token string) ([]byte, error), +) ([]byte, error) { start := 0 pre := []byte(shortcodePlaceholderPrefix) @@ -691,8 +723,11 @@ func replaceShortcodeTokens(source []byte, replacements map[string]string) ([]by } end := j + postIdx + 4 - - newVal := []byte(replacements[string(source[j:end])]) + key := string(source[j:end]) + newVal, err := tokenHandler(ctx, key) + if err != nil { + return nil, err + } // Issue #1148: Check for wrapping p-tags

if j >= 3 && bytes.Equal(source[j-3:j], pStart) { @@ -712,11 +747,11 @@ func replaceShortcodeTokens(source []byte, replacements map[string]string) ([]by return source, nil } -func renderShortcodeWithPage(h tpl.TemplateHandler, tmpl tpl.Template, data *ShortcodeWithPage) (string, error) { +func renderShortcodeWithPage(ctx context.Context, h tpl.TemplateHandler, tmpl tpl.Template, data *ShortcodeWithPage) (string, error) { buffer := bp.GetBuffer() defer bp.PutBuffer(buffer) - err := h.Execute(tmpl, buffer, data) + err := h.ExecuteWithContext(ctx, tmpl, buffer, data) if err != nil { return "", fmt.Errorf("failed to process shortcode: %w", err) } diff --git a/hugolib/shortcode_page.go b/hugolib/shortcode_page.go index 5a56e434f2f..3bc061bc084 100644 --- a/hugolib/shortcode_page.go +++ b/hugolib/shortcode_page.go @@ -14,13 +14,48 @@ package hugolib import ( + "context" "html/template" "github.com/gohugoio/hugo/resources/page" ) +// A placeholder for the TableOfContents markup. This is what we pass to the Goldmark etc. renderers. var tocShortcodePlaceholder = createShortcodePlaceholder("TOC", 0) +// shortcodeRenderer is typically used to delay rendering of inner shortcodes +// marked with placeholders in the content. +type shortcodeRenderer interface { + renderShortcode(context.Context) ([]byte, bool, error) + renderShortcodeString(context.Context) (string, bool, error) +} + +type shortcodeRenderFunc func(context.Context) ([]byte, bool, error) + +func (f shortcodeRenderFunc) renderShortcode(ctx context.Context) ([]byte, bool, error) { + return f(ctx) +} + +func (f shortcodeRenderFunc) renderShortcodeString(ctx context.Context) (string, bool, error) { + b, has, err := f(ctx) + return string(b), has, err +} + +type prerenderedShortcode struct { + s string + hasVariants bool +} + +func (p prerenderedShortcode) renderShortcode(context.Context) ([]byte, bool, error) { + return []byte(p.s), p.hasVariants, nil +} + +func (p prerenderedShortcode) renderShortcodeString(context.Context) (string, bool, error) { + return p.s, p.hasVariants, nil +} + +var zeroShortcode = prerenderedShortcode{} + // This is sent to the shortcodes. They cannot access the content // they're a part of. It would cause an infinite regress. // @@ -50,7 +85,11 @@ func (p *pageForShortcode) page() page.Page { return p.PageWithoutContent.(page.Page) } -func (p *pageForShortcode) TableOfContents() template.HTML { +func (p *pageForShortcode) String() string { + return p.p.String() +} + +func (p *pageForShortcode) TableOfContents(context.Context) template.HTML { p.p.enablePlaceholders() return p.toc } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index b5f27d6218c..2f285d0da4d 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "path/filepath" "reflect" @@ -247,7 +248,7 @@ CSV: {{< myShort >}} func BenchmarkReplaceShortcodeTokens(b *testing.B) { type input struct { in []byte - replacements map[string]string + tokenHandler func(ctx context.Context, token string) ([]byte, error) expect []byte } @@ -263,22 +264,30 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) { {strings.Repeat("A ", 3000) + " HAHAHUGOSHORTCODE-1HBHB." + strings.Repeat("BC ", 1000) + " HAHAHUGOSHORTCODE-1HBHB.", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "Hello World"}, []byte(strings.Repeat("A ", 3000) + " Hello World." + strings.Repeat("BC ", 1000) + " Hello World.")}, } - in := make([]input, b.N*len(data)) cnt := 0 + in := make([]input, b.N*len(data)) for i := 0; i < b.N; i++ { for _, this := range data { - in[cnt] = input{[]byte(this.input), this.replacements, this.expect} + replacements := make(map[string]shortcodeRenderer) + for k, v := range this.replacements { + replacements[k] = prerenderedShortcode{s: v} + } + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + return []byte(this.replacements[token]), nil + } + in[cnt] = input{[]byte(this.input), tokenHandler, this.expect} cnt++ } } b.ResetTimer() cnt = 0 + ctx := context.Background() for i := 0; i < b.N; i++ { for j := range data { currIn := in[cnt] cnt++ - results, err := replaceShortcodeTokens(currIn.in, currIn.replacements) + results, err := expandShortcodeTokens(ctx, currIn.in, currIn.tokenHandler) if err != nil { b.Fatalf("[%d] failed: %s", i, err) continue @@ -383,7 +392,16 @@ func TestReplaceShortcodeTokens(t *testing.T) { }, } { - results, err := replaceShortcodeTokens([]byte(this.input), this.replacements) + replacements := make(map[string]shortcodeRenderer) + for k, v := range this.replacements { + replacements[k] = prerenderedShortcode{s: v} + } + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + return []byte(this.replacements[token]), nil + } + + ctx := context.Background() + results, err := expandShortcodeTokens(ctx, []byte(this.input), tokenHandler) if b, ok := this.expect.(bool); ok && !b { if err == nil { diff --git a/hugolib/site.go b/hugolib/site.go index 0ca7a81b448..e90fa41ff63 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "html/template" "io" @@ -173,7 +174,7 @@ type Site struct { } func (s *Site) Taxonomies() page.TaxonomyList { - s.init.taxonomies.Do() + s.init.taxonomies.Do(context.Background()) return s.taxonomies } @@ -214,8 +215,9 @@ func (init *siteInit) Reset() { init.taxonomies.Reset() } -func (s *Site) initInit(init *lazy.Init, pctx pageContext) bool { - _, err := init.Do() +func (s *Site) initInit(ctx context.Context, init *lazy.Init, pctx pageContext) bool { + _, err := init.Do(ctx) + if err != nil { s.h.FatalError(pctx.wrapError(err)) } @@ -227,7 +229,7 @@ func (s *Site) prepareInits() { var init lazy.Init - s.init.prevNext = init.Branch(func() (any, error) { + s.init.prevNext = init.Branch(func(context.Context) (any, error) { regularPages := s.RegularPages() for i, p := range regularPages { np, ok := p.(nextPrevProvider) @@ -254,7 +256,7 @@ func (s *Site) prepareInits() { return nil, nil }) - s.init.prevNextInSection = init.Branch(func() (any, error) { + s.init.prevNextInSection = init.Branch(func(context.Context) (any, error) { var sections page.Pages s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(pageMapQuery{Prefix: s.home.treeRef.key}, func(n *contentNode) { sections = append(sections, n.p) @@ -311,12 +313,12 @@ func (s *Site) prepareInits() { return nil, nil }) - s.init.menus = init.Branch(func() (any, error) { + s.init.menus = init.Branch(func(context.Context) (any, error) { s.assembleMenus() return nil, nil }) - s.init.taxonomies = init.Branch(func() (any, error) { + s.init.taxonomies = init.Branch(func(context.Context) (any, error) { err := s.pageMap.assembleTaxonomies() return nil, err }) @@ -327,7 +329,7 @@ type siteRenderingContext struct { } func (s *Site) Menus() navigation.Menus { - s.init.menus.Do() + s.init.menus.Do(context.Background()) return s.menus } @@ -1821,7 +1823,9 @@ func (s *Site) renderForTemplate(name, outputFormat string, d any, w io.Writer, return nil } - if err = s.Tmpl().Execute(templ, w, d); err != nil { + ctx := context.Background() + + if err = s.Tmpl().ExecuteWithContext(ctx, templ, w, d); err != nil { return fmt.Errorf("render of %q failed: %w", name, err) } return diff --git a/hugolib/site_render.go b/hugolib/site_render.go index b572c443e1b..51d638ddef0 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -19,9 +19,8 @@ import ( "strings" "sync" - "github.com/gohugoio/hugo/tpl" - "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/tpl" "errors" diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 8dac8fc92bd..a2ee5699478 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -630,7 +631,7 @@ func TestOrderedPages(t *testing.T) { t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Three", rbypubdate[0].Title()) } - bylength := s.RegularPages().ByLength() + bylength := s.RegularPages().ByLength(context.Background()) if bylength[0].Title() != "One" { t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bylength[0].Title()) } @@ -662,7 +663,7 @@ func TestGroupedPages(t *testing.T) { writeSourcesToSource(t, "content", fs, groupedSources...) s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - rbysection, err := s.RegularPages().GroupBy("Section", "desc") + rbysection, err := s.RegularPages().GroupBy(context.Background(), "Section", "desc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -683,7 +684,7 @@ func TestGroupedPages(t *testing.T) { t.Errorf("PageGroup has unexpected number of pages. Third group should have '%d' pages, got '%d' pages", 2, len(rbysection[2].Pages)) } - bytype, err := s.RegularPages().GroupBy("Type", "asc") + bytype, err := s.RegularPages().GroupBy(context.Background(), "Type", "asc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index ca74e9340e2..89255c695ee 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -2,6 +2,7 @@ package hugolib import ( "bytes" + "context" "fmt" "image/jpeg" "io" @@ -1005,7 +1006,7 @@ func getPage(in page.Page, ref string) page.Page { } func content(c resource.ContentProvider) string { - cc, err := c.Content() + cc, err := c.Content(context.Background()) if err != nil { panic(err) } diff --git a/lazy/init.go b/lazy/init.go index b998d0305d3..4de2a83f74f 100644 --- a/lazy/init.go +++ b/lazy/init.go @@ -29,7 +29,7 @@ func New() *Init { // Init holds a graph of lazily initialized dependencies. type Init struct { - // Used in tests + // Used mainly for testing. initCount uint64 mu sync.Mutex @@ -40,11 +40,11 @@ type Init struct { init onceMore out any err error - f func() (any, error) + f func(context.Context) (any, error) } // Add adds a func as a new child dependency. -func (ini *Init) Add(initFn func() (any, error)) *Init { +func (ini *Init) Add(initFn func(context.Context) (any, error)) *Init { if ini == nil { ini = New() } @@ -59,14 +59,14 @@ func (ini *Init) InitCount() int { // AddWithTimeout is same as Add, but with a timeout that aborts initialization. func (ini *Init) AddWithTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) *Init { - return ini.Add(func() (any, error) { - return ini.withTimeout(timeout, f) + return ini.Add(func(ctx context.Context) (any, error) { + return ini.withTimeout(ctx, timeout, f) }) } // Branch creates a new dependency branch based on an existing and adds // the given dependency as a child. -func (ini *Init) Branch(initFn func() (any, error)) *Init { +func (ini *Init) Branch(initFn func(context.Context) (any, error)) *Init { if ini == nil { ini = New() } @@ -75,13 +75,13 @@ func (ini *Init) Branch(initFn func() (any, error)) *Init { // BranchdWithTimeout is same as Branch, but with a timeout. func (ini *Init) BranchWithTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) *Init { - return ini.Branch(func() (any, error) { - return ini.withTimeout(timeout, f) + return ini.Branch(func(ctx context.Context) (any, error) { + return ini.withTimeout(ctx, timeout, f) }) } // Do initializes the entire dependency graph. -func (ini *Init) Do() (any, error) { +func (ini *Init) Do(ctx context.Context) (any, error) { if ini == nil { panic("init is nil") } @@ -92,7 +92,7 @@ func (ini *Init) Do() (any, error) { if prev != nil { // A branch. Initialize the ancestors. if prev.shouldInitialize() { - _, err := prev.Do() + _, err := prev.Do(ctx) if err != nil { ini.err = err return @@ -105,12 +105,12 @@ func (ini *Init) Do() (any, error) { } if ini.f != nil { - ini.out, ini.err = ini.f() + ini.out, ini.err = ini.f(ctx) } for _, child := range ini.children { if child.shouldInitialize() { - _, err := child.Do() + _, err := child.Do(ctx) if err != nil { ini.err = err return @@ -154,7 +154,7 @@ func (ini *Init) Reset() { } } -func (ini *Init) add(branch bool, initFn func() (any, error)) *Init { +func (ini *Init) add(branch bool, initFn func(context.Context) (any, error)) *Init { ini.mu.Lock() defer ini.mu.Unlock() @@ -179,8 +179,8 @@ func (ini *Init) checkDone() { } } -func (ini *Init) withTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) (any, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) +func (ini *Init) withTimeout(ctx context.Context, timeout time.Duration, f func(ctx context.Context) (any, error)) (any, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() c := make(chan verr, 1) diff --git a/lazy/init_test.go b/lazy/init_test.go index 4d871b93733..499ea2cce7b 100644 --- a/lazy/init_test.go +++ b/lazy/init_test.go @@ -48,16 +48,16 @@ func TestInit(t *testing.T) { var result string - f1 := func(name string) func() (any, error) { - return func() (any, error) { + f1 := func(name string) func(context.Context) (any, error) { + return func(context.Context) (any, error) { result += name + "|" doWork() return name, nil } } - f2 := func() func() (any, error) { - return func() (any, error) { + f2 := func() func(context.Context) (any, error) { + return func(context.Context) (any, error) { doWork() return nil, nil } @@ -75,6 +75,8 @@ func TestInit(t *testing.T) { var wg sync.WaitGroup + ctx := context.Background() + // Add some concurrency and randomness to verify thread safety and // init order. for i := 0; i < 100; i++ { @@ -83,20 +85,20 @@ func TestInit(t *testing.T) { defer wg.Done() var err error if rnd.Intn(10) < 5 { - _, err = root.Do() + _, err = root.Do(ctx) c.Assert(err, qt.IsNil) } // Add a new branch on the fly. if rnd.Intn(10) > 5 { branch := branch1_2.Branch(f2()) - _, err = branch.Do() + _, err = branch.Do(ctx) c.Assert(err, qt.IsNil) } else { - _, err = branch1_2_1.Do() + _, err = branch1_2_1.Do(ctx) c.Assert(err, qt.IsNil) } - _, err = branch1_2.Do() + _, err = branch1_2.Do(ctx) c.Assert(err, qt.IsNil) }(i) @@ -114,7 +116,7 @@ func TestInitAddWithTimeout(t *testing.T) { return nil, nil }) - _, err := init.Do() + _, err := init.Do(context.Background()) c.Assert(err, qt.IsNil) } @@ -133,7 +135,7 @@ func TestInitAddWithTimeoutTimeout(t *testing.T) { return nil, nil }) - _, err := init.Do() + _, err := init.Do(context.Background()) c.Assert(err, qt.Not(qt.IsNil)) @@ -149,7 +151,7 @@ func TestInitAddWithTimeoutError(t *testing.T) { return nil, errors.New("failed") }) - _, err := init.Do() + _, err := init.Do(context.Background()) c.Assert(err, qt.Not(qt.IsNil)) } @@ -178,8 +180,8 @@ func TestInitBranchOrder(t *testing.T) { base := New() - work := func(size int, f func()) func() (any, error) { - return func() (any, error) { + work := func(size int, f func()) func(context.Context) (any, error) { + return func(context.Context) (any, error) { doWorkOfSize(size) if f != nil { f() @@ -205,13 +207,14 @@ func TestInitBranchOrder(t *testing.T) { } var wg sync.WaitGroup + ctx := context.Background() for _, v := range inits { v := v wg.Add(1) go func() { defer wg.Done() - _, err := v.Do() + _, err := v.Do(ctx) c.Assert(err, qt.IsNil) }() } @@ -225,17 +228,17 @@ func TestInitBranchOrder(t *testing.T) { func TestResetError(t *testing.T) { c := qt.New(t) r := false - i := New().Add(func() (any, error) { + i := New().Add(func(context.Context) (any, error) { if r { return nil, nil } return nil, errors.New("r is false") }) - _, err := i.Do() + _, err := i.Do(context.Background()) c.Assert(err, qt.IsNotNil) i.Reset() r = true - _, err = i.Do() + _, err = i.Do(context.Background()) c.Assert(err, qt.IsNil) } diff --git a/markup/asciidocext/convert.go b/markup/asciidocext/convert.go index 4c83e0e95b6..c9524778f6b 100644 --- a/markup/asciidocext/convert.go +++ b/markup/asciidocext/convert.go @@ -53,10 +53,10 @@ func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) type asciidocResult struct { converter.Result - toc tableofcontents.Root + toc *tableofcontents.Fragments } -func (r asciidocResult) TableOfContents() tableofcontents.Root { +func (r asciidocResult) TableOfContents() *tableofcontents.Fragments { return r.toc } @@ -205,16 +205,16 @@ func hasAsciiDoc() bool { // extractTOC extracts the toc from the given src html. // It returns the html without the TOC, and the TOC data -func (a *asciidocConverter) extractTOC(src []byte) ([]byte, tableofcontents.Root, error) { +func (a *asciidocConverter) extractTOC(src []byte) ([]byte, *tableofcontents.Fragments, error) { var buf bytes.Buffer buf.Write(src) node, err := html.Parse(&buf) if err != nil { - return nil, tableofcontents.Root{}, err + return nil, nil, err } var ( f func(*html.Node) bool - toc tableofcontents.Root + toc *tableofcontents.Fragments toVisit []*html.Node ) f = func(n *html.Node) bool { @@ -242,12 +242,12 @@ func (a *asciidocConverter) extractTOC(src []byte) ([]byte, tableofcontents.Root } f(node) if err != nil { - return nil, tableofcontents.Root{}, err + return nil, nil, err } buf.Reset() err = html.Render(&buf, node) if err != nil { - return nil, tableofcontents.Root{}, err + return nil, nil, err } // ltrim and rtrim which are added by html.Render res := buf.Bytes()[25:] @@ -256,9 +256,9 @@ func (a *asciidocConverter) extractTOC(src []byte) ([]byte, tableofcontents.Root } // parseTOC returns a TOC root from the given toc Node -func parseTOC(doc *html.Node) tableofcontents.Root { +func parseTOC(doc *html.Node) *tableofcontents.Fragments { var ( - toc tableofcontents.Root + toc tableofcontents.Builder f func(*html.Node, int, int) ) f = func(n *html.Node, row, level int) { @@ -276,9 +276,9 @@ func parseTOC(doc *html.Node) tableofcontents.Root { continue } href := attr(c, "href")[1:] - toc.AddAt(tableofcontents.Heading{ - Text: nodeContent(c), - ID: href, + toc.AddAt(&tableofcontents.Heading{ + Title: nodeContent(c), + ID: href, }, row, level) } f(n.FirstChild, row, level) @@ -289,7 +289,7 @@ func parseTOC(doc *html.Node) tableofcontents.Root { } } f(doc.FirstChild, -1, 0) - return toc + return toc.Build() } func attr(node *html.Node, key string) string { diff --git a/markup/asciidocext/convert_test.go b/markup/asciidocext/convert_test.go index 3a350c5cea6..47208c06616 100644 --- a/markup/asciidocext/convert_test.go +++ b/markup/asciidocext/convert_test.go @@ -21,13 +21,13 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/markup_config" - "github.com/gohugoio/hugo/markup/tableofcontents" qt "github.com/frankban/quicktest" ) @@ -343,49 +343,8 @@ testContent c.Assert(err, qt.IsNil) toc, ok := r.(converter.TableOfContentsProvider) c.Assert(ok, qt.Equals, true) - expected := tableofcontents.Root{ - Headings: tableofcontents.Headings{ - { - ID: "", - Text: "", - Headings: tableofcontents.Headings{ - { - ID: "_introduction", - Text: "Introduction", - Headings: nil, - }, - { - ID: "_section_1", - Text: "Section 1", - Headings: tableofcontents.Headings{ - { - ID: "_section_1_1", - Text: "Section 1.1", - Headings: tableofcontents.Headings{ - { - ID: "_section_1_1_1", - Text: "Section 1.1.1", - Headings: nil, - }, - }, - }, - { - ID: "_section_1_2", - Text: "Section 1.2", - Headings: nil, - }, - }, - }, - { - ID: "_section_2", - Text: "Section 2", - Headings: nil, - }, - }, - }, - }, - } - c.Assert(toc.TableOfContents(), qt.DeepEquals, expected) + + c.Assert(toc.TableOfContents().Identifiers, qt.DeepEquals, collections.SortedStringSlice{"_introduction", "_section_1", "_section_1_1", "_section_1_1_1", "_section_1_2", "_section_2"}) c.Assert(string(r.Bytes()), qt.Not(qt.Contains), "

") } @@ -404,22 +363,7 @@ func TestTableOfContentsWithCode(t *testing.T) { c.Assert(err, qt.IsNil) toc, ok := r.(converter.TableOfContentsProvider) c.Assert(ok, qt.Equals, true) - expected := tableofcontents.Root{ - Headings: tableofcontents.Headings{ - { - ID: "", - Text: "", - Headings: tableofcontents.Headings{ - { - ID: "_some_code_in_the_title", - Text: "Some code in the title", - Headings: nil, - }, - }, - }, - }, - } - c.Assert(toc.TableOfContents(), qt.DeepEquals, expected) + c.Assert(toc.TableOfContents().HeadingsMap["_some_code_in_the_title"].Title, qt.Equals, "Some code in the title") c.Assert(string(r.Bytes()), qt.Not(qt.Contains), "
") } @@ -443,21 +387,7 @@ func TestTableOfContentsPreserveTOC(t *testing.T) { c.Assert(err, qt.IsNil) toc, ok := r.(converter.TableOfContentsProvider) c.Assert(ok, qt.Equals, true) - expected := tableofcontents.Root{ - Headings: tableofcontents.Headings{ - { - ID: "", - Text: "", - Headings: tableofcontents.Headings{ - { - ID: "some-title", - Text: "Some title", - Headings: nil, - }, - }, - }, - }, - } - c.Assert(toc.TableOfContents(), qt.DeepEquals, expected) + + c.Assert(toc.TableOfContents().Identifiers, qt.DeepEquals, collections.SortedStringSlice{"some-title"}) c.Assert(string(r.Bytes()), qt.Contains, "
") } diff --git a/markup/converter/converter.go b/markup/converter/converter.go index c760381f405..7e5b56b0758 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -101,7 +101,7 @@ type DocumentInfo interface { // TableOfContentsProvider provides the content as a ToC structure. type TableOfContentsProvider interface { - TableOfContents() tableofcontents.Root + TableOfContents() *tableofcontents.Fragments } // AnchorNameSanitizer tells how a converter sanitizes anchor names. diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go index a179cd2339a..6c1c7ad0a7c 100644 --- a/markup/goldmark/convert.go +++ b/markup/goldmark/convert.go @@ -160,11 +160,11 @@ var _ identity.IdentitiesProvider = (*converterResult)(nil) type converterResult struct { converter.Result - toc tableofcontents.Root + toc *tableofcontents.Fragments ids identity.Identities } -func (c converterResult) TableOfContents() tableofcontents.Root { +func (c converterResult) TableOfContents() *tableofcontents.Fragments { return c.toc } @@ -228,9 +228,9 @@ type parserContext struct { parser.Context } -func (p *parserContext) TableOfContents() tableofcontents.Root { +func (p *parserContext) TableOfContents() *tableofcontents.Fragments { if v := p.Get(tocResultKey); v != nil { - return v.(tableofcontents.Root) + return v.(*tableofcontents.Fragments) } - return tableofcontents.Root{} + return nil } diff --git a/markup/goldmark/toc.go b/markup/goldmark/toc.go index 396c1d07122..ac5040e8537 100644 --- a/markup/goldmark/toc.go +++ b/markup/goldmark/toc.go @@ -41,8 +41,8 @@ func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parse } var ( - toc tableofcontents.Root - tocHeading tableofcontents.Heading + toc tableofcontents.Builder + tocHeading = &tableofcontents.Heading{} level int row = -1 inHeading bool @@ -53,10 +53,10 @@ func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parse s := ast.WalkStatus(ast.WalkContinue) if n.Kind() == ast.KindHeading { if inHeading && !entering { - tocHeading.Text = headingText.String() + tocHeading.Title = headingText.String() headingText.Reset() toc.AddAt(tocHeading, row, level-1) - tocHeading = tableofcontents.Heading{} + tocHeading = &tableofcontents.Heading{} inHeading = false return s, nil } @@ -106,7 +106,7 @@ func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parse return s, nil }) - pc.Set(tocResultKey, toc) + pc.Set(tocResultKey, toc.Build()) } type tocExtension struct { diff --git a/markup/tableofcontents/tableofcontents.go b/markup/tableofcontents/tableofcontents.go index 2e7f47d207f..bd0aaa8012e 100644 --- a/markup/tableofcontents/tableofcontents.go +++ b/markup/tableofcontents/tableofcontents.go @@ -14,35 +14,104 @@ package tableofcontents import ( + "sort" "strings" + + "github.com/gohugoio/hugo/common/collections" ) +// Empty is an empty ToC. +var Empty = &Fragments{ + Headings: Headings{}, + HeadingsMap: map[string]*Heading{}, +} + +// Builder is used to build the ToC data structure. +type Builder struct { + toc *Fragments +} + +// Add adds the heading to the ToC. +func (b *Builder) AddAt(h *Heading, row, level int) { + if b.toc == nil { + b.toc = &Fragments{} + } + b.toc.addAt(h, row, level) +} + +// Build returns the ToC. +func (b Builder) Build() *Fragments { + if b.toc == nil { + return Empty + } + b.toc.HeadingsMap = make(map[string]*Heading) + b.toc.walk(func(h *Heading) { + if h.ID != "" { + b.toc.HeadingsMap[h.ID] = h + b.toc.Identifiers = append(b.toc.Identifiers, h.ID) + } + }) + sort.Strings(b.toc.Identifiers) + return b.toc +} + // Headings holds the top level headings. -type Headings []Heading +type Headings []*Heading + +// FilterBy returns a new Headings slice with all headings that matches the given predicate. +// For internal use only. +func (h Headings) FilterBy(fn func(*Heading) bool) Headings { + var out Headings + + for _, h := range h { + h.walk(func(h *Heading) { + if fn(h) { + out = append(out, h) + } + }) + } + return out +} // Heading holds the data about a heading and its children. type Heading struct { - ID string - Text string + ID string + Title string Headings Headings } // IsZero is true when no ID or Text is set. func (h Heading) IsZero() bool { - return h.ID == "" && h.Text == "" + return h.ID == "" && h.Title == "" +} + +func (h *Heading) walk(fn func(*Heading)) { + fn(h) + for _, h := range h.Headings { + h.walk(fn) + } } -// Root implements AddAt, which can be used to build the -// data structure for the ToC. -type Root struct { +// Fragments holds the table of contents for a page. +type Fragments struct { + // Headings holds the top level headings. Headings Headings + + // Identifiers holds all the identifiers in the ToC as a sorted slice. + // Note that collections.SortedStringSlice has both a Contains and Count method + // that can be used to identify missing and duplicate IDs. + Identifiers collections.SortedStringSlice + + // HeadingsMap holds all the headings in the ToC as a map. + // Note that with duplicate IDs, the last one will win. + HeadingsMap map[string]*Heading } -// AddAt adds the heading into the given location. -func (toc *Root) AddAt(h Heading, row, level int) { +// addAt adds the heading into the given location. +func (toc *Fragments) addAt(h *Heading, row, level int) { for i := len(toc.Headings); i <= row; i++ { - toc.Headings = append(toc.Headings, Heading{}) + toc.Headings = append(toc.Headings, &Heading{}) } if level == 0 { @@ -50,19 +119,22 @@ func (toc *Root) AddAt(h Heading, row, level int) { return } - heading := &toc.Headings[row] + heading := toc.Headings[row] for i := 1; i < level; i++ { if len(heading.Headings) == 0 { - heading.Headings = append(heading.Headings, Heading{}) + heading.Headings = append(heading.Headings, &Heading{}) } - heading = &heading.Headings[len(heading.Headings)-1] + heading = heading.Headings[len(heading.Headings)-1] } heading.Headings = append(heading.Headings, h) } // ToHTML renders the ToC as HTML. -func (toc Root) ToHTML(startLevel, stopLevel int, ordered bool) string { +func (toc *Fragments) ToHTML(startLevel, stopLevel int, ordered bool) string { + if toc == nil { + return "" + } b := &tocBuilder{ s: strings.Builder{}, h: toc.Headings, @@ -74,6 +146,12 @@ func (toc Root) ToHTML(startLevel, stopLevel int, ordered bool) string { return b.s.String() } +func (toc Fragments) walk(fn func(*Heading)) { + for _, h := range toc.Headings { + h.walk(fn) + } +} + type tocBuilder struct { s strings.Builder h Headings @@ -133,11 +211,11 @@ func (b *tocBuilder) writeHeadings(level, indent int, h Headings) { } } -func (b *tocBuilder) writeHeading(level, indent int, h Heading) { +func (b *tocBuilder) writeHeading(level, indent int, h *Heading) { b.indent(indent) b.s.WriteString("
  • ") if !h.IsZero() { - b.s.WriteString("" + h.Text + "") + b.s.WriteString("" + h.Title + "") } b.writeHeadings(level, indent, h.Headings) b.s.WriteString("
    • \n") diff --git a/markup/tableofcontents/tableofcontents_test.go b/markup/tableofcontents/tableofcontents_test.go index daeb9f991e9..adbda4b00e4 100644 --- a/markup/tableofcontents/tableofcontents_test.go +++ b/markup/tableofcontents/tableofcontents_test.go @@ -17,18 +17,33 @@ import ( "testing" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/collections" ) +var newTestTocBuilder = func() Builder { + var b Builder + b.AddAt(&Heading{Title: "Heading 1", ID: "h1-1"}, 0, 0) + b.AddAt(&Heading{Title: "1-H2-1", ID: "1-h2-1"}, 0, 1) + b.AddAt(&Heading{Title: "1-H2-2", ID: "1-h2-2"}, 0, 1) + b.AddAt(&Heading{Title: "1-H3-1", ID: "1-h2-2"}, 0, 2) + b.AddAt(&Heading{Title: "Heading 2", ID: "h1-2"}, 1, 0) + return b +} + +var newTestToc = func() *Fragments { + return newTestTocBuilder().Build() +} + func TestToc(t *testing.T) { c := qt.New(t) - toc := &Root{} + toc := &Fragments{} - toc.AddAt(Heading{Text: "Heading 1", ID: "h1-1"}, 0, 0) - toc.AddAt(Heading{Text: "1-H2-1", ID: "1-h2-1"}, 0, 1) - toc.AddAt(Heading{Text: "1-H2-2", ID: "1-h2-2"}, 0, 1) - toc.AddAt(Heading{Text: "1-H3-1", ID: "1-h2-2"}, 0, 2) - toc.AddAt(Heading{Text: "Heading 2", ID: "h1-2"}, 1, 0) + toc.addAt(&Heading{Title: "Heading 1", ID: "h1-1"}, 0, 0) + toc.addAt(&Heading{Title: "1-H2-1", ID: "1-h2-1"}, 0, 1) + toc.addAt(&Heading{Title: "1-H2-2", ID: "1-h2-2"}, 0, 1) + toc.addAt(&Heading{Title: "1-H3-1", ID: "1-h2-2"}, 0, 2) + toc.addAt(&Heading{Title: "Heading 2", ID: "h1-2"}, 1, 0) got := toc.ToHTML(1, -1, false) c.Assert(got, qt.Equals, `