diff --git a/deps/deps.go b/deps/deps.go index ecbba2e5619..557f036f1fa 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -286,6 +286,10 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er return nil, err } + if err != nil { + return nil, err + } + d.Site = cfg.Site // The resource cache is global so reuse. diff --git a/docs/content/en/content-management/formats.md b/docs/content/en/content-management/formats.md index ea056861657..3704ed8b071 100644 --- a/docs/content/en/content-management/formats.md +++ b/docs/content/en/content-management/formats.md @@ -23,7 +23,13 @@ You can put any file type into your `/content` directories, but Hugo uses the `m * [Shortcodes](/content-management/shortcodes/) processed * Layout applied -## List of content formats +{{< deleteme >}} + + +## List of content formats. + + + The current list of content formats in Hugo: diff --git a/docs/content/en/getting-started/quick-start.md b/docs/content/en/getting-started/quick-start.md index 143dc0a4172..2cabdfbb756 100644 --- a/docs/content/en/getting-started/quick-start.md +++ b/docs/content/en/getting-started/quick-start.md @@ -24,8 +24,11 @@ This quick start uses `macOS` in the examples. For instructions about how to ins It is recommended to have [Git](https://git-scm.com/downloads) installed to run this tutorial. {{% /note %}} +![Drag Racing](/images/Dragster.jpg "image title") +![Drag Racing](/images/Dragster2.jpg "image title") + ## Step 1: Install Hugo {{% note %}} diff --git a/docs/layouts/_default/_markup/render-image.html b/docs/layouts/_default/_markup/render-image.html new file mode 100644 index 00000000000..f4b9661dfc8 --- /dev/null +++ b/docs/layouts/_default/_markup/render-image.html @@ -0,0 +1,2 @@ +{{ $url := "https://upload.wikimedia.org/wikipedia/commons/thumb/6/6a/Mating_pair_of_Castalius_rosimon_WLB_DSC_2823.jpg/1024px-Mating_pair_of_Castalius_rosimon_WLB_DSC_2823.jpg" | safeURL }} + \ No newline at end of file diff --git a/docs/layouts/_default/_markup/render-link.html b/docs/layouts/_default/_markup/render-link.html new file mode 100644 index 00000000000..0df3929f6b6 --- /dev/null +++ b/docs/layouts/_default/_markup/render-link.html @@ -0,0 +1 @@ +😉😉{{ .Text | safeHTML }} 😉😉 \ No newline at end of file diff --git a/docs/layouts/partials/deleteme.html b/docs/layouts/partials/deleteme.html new file mode 100644 index 00000000000..e4df2ce650f --- /dev/null +++ b/docs/layouts/partials/deleteme.html @@ -0,0 +1 @@ +THIS IS PARTIAL!!! \ No newline at end of file diff --git a/docs/layouts/shortcodes/deleteme.html b/docs/layouts/shortcodes/deleteme.html new file mode 100644 index 00000000000..b85b58fc1fc --- /dev/null +++ b/docs/layouts/shortcodes/deleteme.html @@ -0,0 +1,3 @@ +DELETEME PARTIAL: + +{{ partial "deleteme" }} \ No newline at end of file diff --git a/helpers/content.go b/helpers/content.go index 4dc4cd413bd..1c780fefe1b 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -25,13 +25,14 @@ import ( "github.com/gohugoio/hugo/common/loggers" + "github.com/spf13/afero" + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" "strings" ) @@ -78,6 +79,7 @@ func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero ContentFs: contentFs, Logger: logger, }) + if err != nil { return nil, err } diff --git a/helpers/general_test.go b/helpers/general_test.go index 104a4c35def..b45fb0e9b44 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -21,9 +21,8 @@ import ( "github.com/spf13/viper" - "github.com/gohugoio/hugo/common/loggers" - qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/loggers" "github.com/spf13/afero" ) diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go new file mode 100644 index 00000000000..db9ba9f46e8 --- /dev/null +++ b/hugolib/content_render_hooks_test.go @@ -0,0 +1,158 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless requiredF by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import "testing" + +func TestRenderHooks(t *testing.T) { + // TODO1 markdownify + config := ` +baseURL="https://example.org" +workingDir="/mywork" +` + b := newTestSitesBuilder(t).WithWorkingDir("/mywork").WithConfigFile("toml", config).Running() + b.WithTemplatesAdded("_default/single.html", `{{ .Content }}`) + b.WithTemplatesAdded("shortcodes/myshortcode1.html", `{{ partial "mypartial1" }}`) + b.WithTemplatesAdded("shortcodes/myshortcode2.html", `{{ partial "mypartial2" }}`) + b.WithTemplatesAdded("shortcodes/myshortcode3.html", `SHORT3|`) + b.WithTemplatesAdded("shortcodes/myshortcode4.html", ` +
+{{ .Inner | markdownify }} +
+`) + b.WithTemplatesAdded("shortcodes/myshortcode5.html", ` +
+{{ .Inner | .Page.RenderString }} +
+`) + + b.WithTemplatesAdded("shortcodes/myshortcode6.html", `.Render: {{ .Page.Render "myrender" }}`) + b.WithTemplatesAdded("partials/mypartial1.html", `PARTIAL1`) + b.WithTemplatesAdded("partials/mypartial2.html", `PARTIAL2 {{ partial "mypartial3.html" }}`) + b.WithTemplatesAdded("partials/mypartial3.html", `PARTIAL3`) + b.WithTemplatesAdded("partials/mypartial4.html", `PARTIAL4`) + b.WithTemplatesAdded("customview/myrender.html", `myrender: {{ .Title }}|P4: {{ partial "mypartial4" }}`) + b.WithTemplatesAdded("_default/_markup/render-link.html", `{{ with .Page }}{{ .Title }}{{ end }}|{{ .Destination | safeURL }}|Title: {{ .Title | safeHTML }}|Text: {{ .Text | safeHTML }}|END`) + b.WithTemplatesAdded("docs/_markup/render-link.html", `Link docs section: {{ .Text | safeHTML }}|END`) + b.WithTemplatesAdded("_default/_markup/render-image.html", `IMAGE: {{ .Page.Title }}||{{ .Destination | safeURL }}|Title: {{ .Title | safeHTML }}|Text: {{ .Text | safeHTML }}|END`) + + b.WithContent("customview/p1.md", `--- +title: Custom View +--- + +{{< myshortcode6 >}} + + `, "blog/p1.md", `--- +title: Cool Page +--- + +[First Link](https://www.google.com "Google's Homepage") + +{{< myshortcode3 >}} + +[Second Link](https://www.google.com "Google's Homepage") + +Image: + +![Drag Racing](/images/Dragster.jpg "image title") + + +`, "blog/p2.md", `--- +title: Cool Page2 +layout: mylayout +--- + +{{< myshortcode1 >}} + +[Some Text](https://www.google.com "Google's Homepage") + + + +`, "blog/p3.md", `--- +title: Cool Page3 +--- + +{{< myshortcode2 >}} + + +`, "docs/docs1.md", `--- +title: Docs 1 +--- + + +[Docs 1](https://www.google.com "Google's Homepage") + + +`, "blog/p4.md", `--- +title: Cool Page With Image +--- + +Image: + +![Drag Racing](/images/Dragster.jpg "image title") + + +`, "blog/p5.md", `--- +title: Cool Page With Markdownify +--- + +{{< myshortcode4 >}} +Inner Link: [Inner Link](https://www.google.com "Google's Homepage") +{{< /myshortcode4 >}} + +`, "blog/p6.md", `--- +title: With RenderString +--- + +{{< myshortcode5 >}}Inner Link: [Inner Link](https://www.gohugo.io "Hugo's Homepage"){{< /myshortcode5 >}} + +`) + b.Build(BuildCfg{}) + b.AssertFileContent("public/blog/p1/index.html", ` +

Cool Page|https://www.google.com|Title: Google's Homepage|Text: First Link|END

+Text: Second +SHORT3| +

IMAGE: Cool Page||/images/Dragster.jpg|Title: image title|Text: Drag Racing|END

+`) + + b.AssertFileContent("public/customview/p1/index.html", `.Render: myrender: Custom View|P4: PARTIAL4`) + b.AssertFileContent("public/blog/p2/index.html", `PARTIAL`) + b.AssertFileContent("public/blog/p3/index.html", `PARTIAL3`) + b.AssertFileContent("public/docs/docs1/index.html", `Link docs section: Docs 1|END`) + b.AssertFileContent("public/blog/p4/index.html", `

IMAGE: Cool Page With Image||/images/Dragster.jpg|Title: image title|Text: Drag Racing|END

`) + // The regular markdownify func currently gets regular links. + b.AssertFileContent("public/blog/p5/index.html", "Inner Link: Inner Link\n") + + b.AssertFileContent("public/blog/p6/index.html", "
\n

Inner Link: With RenderString|https://www.gohugo.io|Title: Hugo's Homepage|Text: Inner Link|END

\n\n
") + + b.EditFiles( + "layouts/_default/_markup/render-link.html", `EDITED: {{ .Destination | safeURL }}|`, + "layouts/_default/_markup/render-image.html", `IMAGE EDITED: {{ .Destination | safeURL }}|`, + "layouts/docs/_markup/render-link.html", `DOCS EDITED: {{ .Destination | safeURL }}|`, + "layouts/partials/mypartial1.html", `PARTIAL1_EDITED`, + "layouts/partials/mypartial3.html", `PARTIAL3_EDITED`, + "layouts/partials/mypartial4.html", `PARTIAL4_EDITED`, + "layouts/shortcodes/myshortcode3.html", `SHORT3_EDITED|`, + ) + + b.Build(BuildCfg{}) + b.AssertFileContent("public/customview/p1/index.html", `.Render: myrender: Custom View|P4: PARTIAL4_EDITED`) + b.AssertFileContent("public/blog/p1/index.html", `

EDITED: https://www.google.com|

`, "SHORT3_EDITED|") + b.AssertFileContent("public/blog/p2/index.html", `PARTIAL1_EDITED`) + b.AssertFileContent("public/blog/p3/index.html", `PARTIAL3_EDITED`) + b.AssertFileContent("public/docs/docs1/index.html", `DOCS EDITED: https://www.google.com|

`) + b.AssertFileContent("public/blog/p4/index.html", `IMAGE EDITED: /images/Dragster.jpg|`) + b.AssertFileContent("public/blog/p6/index.html", "

Inner Link: EDITED: https://www.gohugo.io|

") + +} diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index de6baa130d7..cdc39ce61cb 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -126,10 +126,28 @@ type SourceFilesystems struct { StaticDirs []hugofs.FileMetaInfo } +// FileSystems returns the FileSystems relevant for the change detection +// in server mode. +// Note: This does currently not return any static fs. +func (s *SourceFilesystems) FileSystems() []*SourceFilesystem { + return []*SourceFilesystem{ + s.Content, + s.Data, + s.I18n, + s.Layouts, + s.Archetypes, + // TODO(bep) static + } + +} + // A SourceFilesystem holds the filesystem for a given source type in Hugo (data, // i18n, layouts, static) and additional metadata to be able to use that filesystem // in server mode. type SourceFilesystem struct { + // Name matches one in files.ComponentFolders + Name string + // This is a virtual composite filesystem. It expects path relative to a context. Fs afero.Fs @@ -275,6 +293,19 @@ func (d *SourceFilesystem) Contains(filename string) bool { return false } +// Path returns the relative path to the given filename if it is a member of +// of the current filesystem, an empty string if not. +func (d *SourceFilesystem) Path(filename string) string { + for _, dir := range d.Dirs { + meta := dir.Meta() + if strings.HasPrefix(filename, meta.Filename()) { + p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator) + return p + } + } + return "" +} + // RealDirs gets a list of absolute paths to directories starting from the given // path. func (d *SourceFilesystem) RealDirs(from string) []string { @@ -349,12 +380,14 @@ func newSourceFilesystemsBuilder(p *paths.Paths, logger *loggers.Logger, b *Base return &sourceFilesystemsBuilder{p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} } -func (b *sourceFilesystemsBuilder) newSourceFilesystem(fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { +func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { return &SourceFilesystem{ + Name: name, Fs: fs, Dirs: dirs, } } + func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { if b.theBigFs == nil { @@ -369,12 +402,12 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { createView := func(componentID string) *SourceFilesystem { if b.theBigFs == nil || b.theBigFs.overlayMounts == nil { - return b.newSourceFilesystem(hugofs.NoOpFs, nil) + return b.newSourceFilesystem(componentID, hugofs.NoOpFs, nil) } dirs := b.theBigFs.overlayDirs[componentID] - return b.newSourceFilesystem(afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) + return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) } @@ -392,14 +425,14 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { return nil, err } - b.result.Data = b.newSourceFilesystem(dataFs, dataDirs) + b.result.Data = b.newSourceFilesystem(files.ComponentFolderData, dataFs, dataDirs) i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n] i18nFs, err := hugofs.NewSliceFs(i18nDirs...) if err != nil { return nil, err } - b.result.I18n = b.newSourceFilesystem(i18nFs, i18nDirs) + b.result.I18n = b.newSourceFilesystem(files.ComponentFolderI18n, i18nFs, i18nDirs) contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent] contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent) @@ -409,7 +442,7 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { return nil, errors.Wrap(err, "create content filesystem") } - b.result.Content = b.newSourceFilesystem(contentFs, contentDirs) + b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs, contentDirs) b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull) @@ -421,13 +454,13 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { if b.theBigFs.staticPerLanguage != nil { // Multihost mode for k, v := range b.theBigFs.staticPerLanguage { - sfs := b.newSourceFilesystem(v, b.result.StaticDirs) + sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v, b.result.StaticDirs) sfs.PublishFolder = k ms[k] = sfs } } else { bfs := afero.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic) - ms[""] = b.newSourceFilesystem(bfs, b.result.StaticDirs) + ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs, b.result.StaticDirs) } return b.result, nil diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 40185e051c2..90044327533 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -40,6 +40,9 @@ import ( // TODO(bep) this fails when testmodBuilder is also building ... func TestHugoModules(t *testing.T) { + if !isCI() { + t.Skip("skip (relative) long running modules test when running locally") + } t.Parallel() if !isCI() || hugo.GoMinorVersion() < 12 { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index c71dcaa5940..a2d91df9266 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -20,6 +20,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/identity" + radix "github.com/armon/go-radix" "github.com/gohugoio/hugo/output" @@ -411,7 +413,6 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { } d.OutputFormatsConfig = s.outputFormatsConfig } - } return nil @@ -806,12 +807,40 @@ func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Page return h.Sites[0].findPagesByKindIn(kind, inPages) } -func (h *HugoSites) findPagesByShortcode(shortcode string) page.Pages { - var pages page.Pages +func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { + for _, s := range h.Sites { - pages = append(pages, s.findPagesByShortcode(shortcode)...) + PAGES: + for _, p := range s.rawAllPages { + OUTPUTS: + for _, po := range p.pageOutputs { + if po.cp == nil { + continue + } + for id, _ := range idset { + if po.cp.dependencyTracker.Search(id) != nil { + po.cp.Reset() + p.forceRender = true + continue OUTPUTS + } + } + } + + for _, s := range p.shortcodeState.shortcodes { + for id, _ := range idset { + if s.info.Search(id) != nil { + for _, po := range p.pageOutputs { + if po.cp != nil { + po.cp.Reset() + } + } + p.forceRender = true + continue PAGES + } + } + } + } } - return pages } // Used in partial reloading to determine if the change is in a bundle. diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index a70a19e7c31..d749ff581d5 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -71,7 +71,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { if conf.whatChanged == nil { // Assume everything has changed - conf.whatChanged = &whatChanged{source: true, other: true} + conf.whatChanged = &whatChanged{source: true} } var prepareErr error diff --git a/hugolib/page.go b/hugolib/page.go index 56202f5e0b1..2df5006e34a 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -23,6 +23,12 @@ import ( "sort" "strings" + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/tpl" + + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/common/maps" @@ -46,6 +52,7 @@ import ( "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/text" + "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" @@ -59,7 +66,11 @@ var ( var ( pageTypesProvider = resource.NewResourceTypesProvider(media.OctetType, pageResourceType) - nopPageOutput = &pageOutput{pagePerOutputProviders: nopPagePerOutput} + nopPageOutput = &pageOutput{ + pagePerOutputProviders: nopPagePerOutput, + ContentProvider: page.NopPage, + TableOfContentsProvider: page.NopPage, + } ) // pageContext provides contextual information about this page, for error @@ -317,6 +328,106 @@ func (ps *pageState) initCommonProviders(pp pagePaths) error { return nil } +// TODO1 remove +func (ps *pageState) initOutputFormats() error { + + if len(ps.pageOutputs) == 0 { + return nil + } + + if true { + return nil + } + + c := ps.getContentConverter() + if c == nil || !c.Supports(converter.FeatureRenderHooks) { + return nil + } + + templSet := make(map[identity.Identity]bool) + canReuse := true + + for _, o := range ps.pageOutputs { + if !o.render || o.cp == nil { + continue + } + h, err := ps.createRenderHooks(o.f) + if err != nil { + return err + } + if h == nil { + continue + } + if canReuse { + for _, r := range []hooks.LinkRenderer{h.LinkRenderer, h.ImageRenderer} { + if !canReuse { + break + } + if r != nil { + if len(templSet) != 0 { + // There may be a template per output format. + // In that case we need to re-render. + canReuse = templSet[r.GetIdentity()] + } + templSet[r.GetIdentity()] = true + } + } + } + o.cp.renderHooks = h + o.cp.renderHooksHaveVariants = !canReuse + } + + return nil +} + +func (p *pageState) createRenderHooks(f output.Format) (*hooks.Render, error) { + + layoutDescriptor := p.getLayoutDescriptor() + layoutDescriptor.RenderingHook = true + layoutDescriptor.LayoutOverride = false + layoutDescriptor.Layout = "" + + layoutDescriptor.Kind = "render-link" + linkLayouts, err := p.s.layoutHandler.For(layoutDescriptor, f) + if err != nil { + return nil, err + } + + layoutDescriptor.Kind = "render-image" + imageLayouts, err := p.s.layoutHandler.For(layoutDescriptor, f) + if err != nil { + return nil, err + } + + if linkLayouts == nil && imageLayouts == nil { + return nil, nil + } + + var linkRenderer hooks.LinkRenderer + var imageRenderer hooks.LinkRenderer + + if templ, found := p.s.lookupTemplate(linkLayouts...); found { + linkRenderer = contentLinkRenderer{ + templateHandler: p.s.Tmpl, + Provider: templ.(tpl.TemplateInfoProvider).TemplateInfo(), + templ: templ, + } + } + + if templ, found := p.s.lookupTemplate(imageLayouts...); found { + imageRenderer = contentLinkRenderer{ + templateHandler: p.s.Tmpl, + Provider: templ.(tpl.TemplateInfoProvider).TemplateInfo(), + templ: templ, + } + } + + return &hooks.Render{ + LinkRenderer: linkRenderer, + ImageRenderer: imageRenderer, + }, nil +} + func (p *pageState) getLayoutDescriptor() output.LayoutDescriptor { p.layoutDescriptorInit.Do(func() { var section string @@ -464,11 +575,40 @@ func (p *pageState) AlternativeOutputFormats() page.OutputFormats { return o } -func (p *pageState) Render(layout ...string) template.HTML { +// TODO1 option: map: display: inline/block (default inline) +func (p *pageState) RenderString(in interface{}) (template.HTML, error) { + s, err := cast.ToStringE(in) + if err != nil { + return "", p.wrapError(err) + } + + b, err := p.pageOutput.cp.renderContent([]byte(s), false) + if err != nil { + return "", p.wrapError(err) + } + + if str, ok := b.(fmt.Stringer); ok { + return template.HTML(str.String()), nil + } + return template.HTML(b.Bytes()), nil +} + +func (p *pageState) addDependency(dep identity.Provider) { + if !p.s.running() || p.pageOutput.cp == nil { + return + } + p.pageOutput.cp.dependencyTracker.Add(dep) +} + +func (p *pageState) RenderWithTemplateInfo(info tpl.TemplateInfo, layout ...string) (template.HTML, error) { + p.addDependency(info.TemplateInfo()) + return p.Render(layout...) +} + +func (p *pageState) Render(layout ...string) (template.HTML, error) { l, err := p.getLayouts(layout...) if err != nil { - p.s.SendError(p.wrapError(errors.Errorf(".Render: failed to resolve layout %v", layout))) - return "" + return "", p.wrapError(errors.Errorf("failed to resolve layout %v", layout)) } for _, layout := range l { @@ -479,17 +619,18 @@ func (p *pageState) Render(layout ...string) template.HTML { // We default to good old HTML. templ, _ = p.s.Tmpl.Lookup(layout + ".html") } + if templ != nil { + p.addDependency(templ.(tpl.TemplateInfoProvider).TemplateInfo()) res, err := executeToString(p.s.Tmpl, templ, p) if err != nil { - p.s.SendError(p.wrapError(errors.Wrapf(err, ".Render: failed to execute template %q v", layout))) - return "" + return "", p.wrapError(errors.Wrapf(err, "failed to execute template %q v", layout)) } - return template.HTML(res) + return template.HTML(res), nil } } - return "" + return "", nil } diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index 1fc69c21826..37e333cb40b 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -592,7 +592,7 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte return nil } -func (p *pageMeta) applyDefaultValues() error { +func (p *pageMeta) applyDefaultValues(ps *pageState) error { if p.markup == "" { if !p.File().IsZero() { // Fall back to file extension @@ -656,15 +656,19 @@ func (p *pageMeta) applyDefaultValues() error { return errors.Errorf("no content renderer found for markup %q", p.markup) } - cpp, err := cp.New(converter.DocumentContext{ - DocumentID: p.f.UniqueID(), - DocumentName: p.f.Path(), - ConfigOverrides: renderingConfigOverrides, - }) + cpp, err := cp.New( + converter.DocumentContext{ + Document: newPageForRenderHook(ps), + DocumentID: p.f.UniqueID(), + DocumentName: p.f.Path(), + ConfigOverrides: renderingConfigOverrides, + }, + ) if err != nil { return err } + p.contentConverter = cpp } diff --git a/hugolib/page__new.go b/hugolib/page__new.go index 99bf305aa58..902f07d867d 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -112,7 +112,7 @@ func newPageFromMeta(meta map[string]interface{}, metaProvider *pageMeta) (*page } } - if err := metaProvider.applyDefaultValues(); err != nil { + if err := metaProvider.applyDefaultValues(ps); err != nil { return err } @@ -134,7 +134,7 @@ func newPageFromMeta(meta map[string]interface{}, metaProvider *pageMeta) (*page } makeOut := func(f output.Format, render bool) *pageOutput { - return newPageOutput(nil, ps, pp, f, render) + return newPageOutput(ps, pp, f, render) } if ps.m.standalone { @@ -158,6 +158,10 @@ func newPageFromMeta(meta map[string]interface{}, metaProvider *pageMeta) (*page return nil, err } + if err := ps.initOutputFormats(); err != nil { + return nil, err + } + return nil, nil }) @@ -234,7 +238,7 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope return ps.wrapError(err) } - if err := metaProvider.applyDefaultValues(); err != nil { + if err := metaProvider.applyDefaultValues(ps); err != nil { return err } @@ -242,6 +246,7 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope } ps.init.Add(func() (interface{}, error) { + // TODO1 reuseContent := ps.renderable && !ps.shortcodeState.hasShortcodes() // Creates what's needed for each output format. @@ -264,18 +269,17 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope } _, render := outputFormatsForPage.GetByName(f.Name) - var contentProvider *pageContentOutput + po := newPageOutput(ps, pp, f, render) if reuseContent && i > 0 { - contentProvider = ps.pageOutputs[0].cp + po.initContentProvider(ps.pageOutputs[0].cp) } else { - var err error - contentProvider, err = contentPerOutput(f) + contentProvider, err := contentPerOutput(po) if err != nil { return nil, err } + po.initContentProvider(contentProvider) } - po := newPageOutput(contentProvider, ps, pp, f, render) ps.pageOutputs[i] = po created[f.Name] = po } @@ -284,6 +288,10 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope return nil, err } + if err := ps.initOutputFormats(); err != nil { + return nil, err + } + return nil, nil }) diff --git a/hugolib/page__output.go b/hugolib/page__output.go index 764c46a937b..0afc6859317 100644 --- a/hugolib/page__output.go +++ b/hugolib/page__output.go @@ -14,13 +14,13 @@ package hugolib import ( + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" ) func newPageOutput( - cp *pageContentOutput, // may be nil ps *pageState, pp pagePaths, f output.Format, @@ -45,36 +45,23 @@ func newPageOutput( paginatorProvider = pag } - var ( - contentProvider page.ContentProvider = page.NopPage - tableOfContentsProvider page.TableOfContentsProvider = page.NopPage - ) - - if cp != nil { - contentProvider = cp - tableOfContentsProvider = cp - } - providers := struct { - page.ContentProvider - page.TableOfContentsProvider page.PaginatorProvider resource.ResourceLinksProvider targetPather }{ - contentProvider, - tableOfContentsProvider, paginatorProvider, linksProvider, targetPathsProvider, } po := &pageOutput{ - f: f, - cp: cp, - pagePerOutputProviders: providers, - render: render, - paginator: pag, + f: f, + pagePerOutputProviders: providers, + ContentProvider: page.NopPage, + TableOfContentsProvider: page.NopPage, + render: render, + paginator: pag, } return po @@ -94,16 +81,54 @@ type pageOutput struct { // used in template(s). paginator *pagePaginator - // This interface provides the functionality that is specific for this + // These interface provides the functionality that is specific for this // output format. pagePerOutputProviders + page.ContentProvider + page.TableOfContentsProvider - // This may be nil. + // May be nil. cp *pageContentOutput } +func (o *pageOutput) initRenderHooks() error { + if !o.render || o.cp == nil { + return nil + } + + ps := o.cp.p + + c := ps.getContentConverter() + if c == nil || !c.Supports(converter.FeatureRenderHooks) { + return nil + } + + h, err := ps.createRenderHooks(o.f) + if err != nil { + return err + } + if h == nil { + return nil + } + + o.cp.renderHooks = h + + return nil + +} + +func (p *pageOutput) initContentProvider(cp *pageContentOutput) { + if cp == nil { + return + } + p.ContentProvider = cp + p.TableOfContentsProvider = cp + p.cp = cp +} + func (p *pageOutput) enablePlaceholders() { if p.cp != nil { p.cp.enablePlaceholders() } + } diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index d3a32e15c2a..669619ce7bc 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -23,6 +23,10 @@ import ( "sync" "unicode/utf8" + "github.com/gohugoio/hugo/identity" + + "github.com/gohugoio/hugo/markup/converter/hooks" + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/lazy" @@ -58,14 +62,22 @@ var ( } ) -func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutput, error) { +var pageContentOutputDependenciesID = identity.KeyValueIdentity{Key: "pageOutput", Value: "dependencies"} + +func newPageContentOutput(p *pageState) func(po *pageOutput) (*pageContentOutput, error) { parent := p.init - return func(f output.Format) (*pageContentOutput, error) { + return func(po *pageOutput) (*pageContentOutput, error) { + var dependencyTracker identity.Manager + if p.s.running() { + dependencyTracker = identity.NewIdentityManager(pageContentOutputDependenciesID) + } + cp := &pageContentOutput{ - p: p, - f: f, + dependencyTracker: dependencyTracker, + p: p, + f: po.f, } initContent := func() (err error) { @@ -81,15 +93,28 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu } }() + if err := po.initRenderHooks(); err != nil { + return err + } + var hasVariants bool + f := po.f cp.contentPlaceholders, hasVariants, err = p.shortcodeState.renderShortcodesForPage(p, f) if err != nil { return err } - if p.render && !hasVariants { - // We can reuse this for the other output formats + enableReuse := po.render && !hasVariants + enableReuse = enableReuse && !cp.renderHooksHaveVariants + + if enableReuse { + // Reuse this for the other output formats. + // We may improve on this, but we really want to avoid re-rendering the content + // to all output formats. + // The current rule is that if you need output format-aware shortcodes or + // content rendering hooks, create a output format-specific template, e.g. + // myshortcode.amp.html. cp.enableReuse() } @@ -99,11 +124,12 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu if p.renderable { if !isHTML { - r, err := cp.renderContent(cp.workContent) + r, err := cp.renderContent(cp.workContent, true) if err != nil { return err } - cp.convertedResult = r + + cp.convertedResult = r // TODO1 avoid storing this cp.workContent = r.Bytes() if _, ok := r.(converter.TableOfContentsProvider); !ok { @@ -150,12 +176,7 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu } } } else if cp.p.m.summary != "" { - b, err := cp.p.getContentConverter().Convert( - converter.RenderContext{ - Src: []byte(cp.p.m.summary), - }, - ) - + b, err := cp.renderContent([]byte(cp.p.m.summary), false) if err != nil { return err } @@ -178,6 +199,7 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu // Recursive loops can only happen in content files with template code (shortcodes etc.) // Avoid creating new goroutines if we don't have to. needTimeout := !p.renderable || p.shortcodeState.hasShortcodes() + needTimeout = needTimeout || cp.renderHooks != nil if needTimeout { cp.initMain = parent.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (interface{}, error) { @@ -211,7 +233,7 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu type pageContentOutput struct { f output.Format - // If we can safely reuse this for other output formats. + // If we can reuse this for other output formats. reuse bool reuseInit sync.Once @@ -224,10 +246,16 @@ type pageContentOutput struct { placeholdersEnabled bool placeholdersEnabledInit sync.Once + // May be nil. + renderHooks *hooks.Render + // Set if there are more than one output format variant + renderHooksHaveVariants bool // TODO1 reimplement this in another way + // Content state - workContent []byte - convertedResult converter.Result + workContent []byte + convertedResult converter.Result + dependencyTracker identity.Manager // Set in server mode. // Temporary storage of placeholders mapped to their content. // These are shortcodes etc. Some of these will need to be replaced @@ -248,6 +276,21 @@ type pageContentOutput struct { readingTime int } +func (p *pageContentOutput) trackDependency(id identity.Provider) { + if p.dependencyTracker != nil { + p.dependencyTracker.Add(id) + } +} + +func (p *pageContentOutput) Reset() { + if p.dependencyTracker != nil { + p.dependencyTracker.Reset() + } + p.p.initOutputFormats() + p.initMain.Reset() + p.initPlain.Reset() +} + func (p *pageContentOutput) Content() (interface{}, error) { if p.p.s.initInit(p.initMain, p.p) { return p.content, nil @@ -331,12 +374,25 @@ func (p *pageContentOutput) setAutoSummary() error { } -func (cp *pageContentOutput) renderContent(content []byte) (converter.Result, error) { - return cp.p.getContentConverter().Convert( +func (cp *pageContentOutput) renderContent(content []byte, renderTOC bool) (converter.Result, error) { + c := cp.p.getContentConverter() + r, err := c.Convert( converter.RenderContext{ - Src: content, - RenderTOC: true, + Src: content, + RenderTOC: renderTOC, + RenderHooks: cp.renderHooks, }) + + if err == nil { + if ids, ok := r.(identity.IdentitiesProvider); ok { + for _, v := range ids.GetIdentities() { + cp.trackDependency(v) + } + } + } + + return r, err + } func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) { @@ -392,9 +448,7 @@ func (p *pageContentOutput) enableReuse() { // these will be shifted out when rendering a given output format. type pagePerOutputProviders interface { targetPather - page.ContentProvider page.PaginatorProvider - page.TableOfContentsProvider resource.ResourceLinksProvider } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index dc8bc821c15..f4bf3ac0040 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -93,12 +93,6 @@ Summary Next Line. {{
}}. More text here. Some more text -` - - simplePageWithEmbeddedScript = `--- -title: Simple ---- - ` simplePageWithSummaryDelimiterSameLine = `--- @@ -325,6 +319,7 @@ func normalizeContent(c string) string { } func checkPageTOC(t *testing.T, page page.Page, toc string) { + t.Helper() if page.TableOfContents() != template.HTML(toc) { t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(), toc) } diff --git a/hugolib/page_unwrap_test.go b/hugolib/page_unwrap_test.go index 20888166ad7..bcc1b769a4f 100644 --- a/hugolib/page_unwrap_test.go +++ b/hugolib/page_unwrap_test.go @@ -26,6 +26,7 @@ func TestUnwrapPage(t *testing.T) { p := &pageState{} c.Assert(mustUnwrap(newPageForShortcode(p)), qt.Equals, p) + c.Assert(mustUnwrap(newPageForRenderHook(p)), qt.Equals, p) } func mustUnwrap(v interface{}) page.Page { diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go index 7e9682e90e1..adcbbccefef 100644 --- a/hugolib/pagecollections.go +++ b/hugolib/pagecollections.go @@ -358,16 +358,6 @@ func (c *PageCollections) removePage(page *pageState) { } } -func (c *PageCollections) findPagesByShortcode(shortcode string) page.Pages { - var pages page.Pages - for _, p := range c.rawAllPages { - if p.HasShortcode(shortcode) { - pages = append(pages, p) - } - } - return pages -} - func (c *PageCollections) replacePage(page *pageState) { // will find existing page that matches filepath and remove it c.removePage(page) diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 69bcb6d4f73..3f0a054e708 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -23,8 +23,6 @@ import ( "html/template" "path" - "github.com/gohugoio/hugo/markup/converter" - "github.com/gohugoio/hugo/common/herrors" "github.com/pkg/errors" @@ -351,12 +349,7 @@ func renderShortcode( // shortcode. if sc.doMarkup && (level > 0 || sc.info.Config.Version == 1) { var err error - - b, err := p.getContentConverter().Convert( - converter.RenderContext{ - Src: []byte(inner), - }, - ) + b, err := p.pageOutput.cp.renderContent([]byte(inner), false) if err != nil { return "", false, err diff --git a/hugolib/shortcode_page.go b/hugolib/shortcode_page.go index e8a3a37e19b..5a56e434f2f 100644 --- a/hugolib/shortcode_page.go +++ b/hugolib/shortcode_page.go @@ -54,3 +54,22 @@ func (p *pageForShortcode) TableOfContents() template.HTML { p.p.enablePlaceholders() return p.toc } + +// This is what is sent into the content render hooks (link, image). +type pageForRenderHooks struct { + page.PageWithoutContent + page.TableOfContentsProvider + page.ContentProvider +} + +func newPageForRenderHook(p *pageState) page.Page { + return &pageForRenderHooks{ + PageWithoutContent: p, + ContentProvider: page.NopPage, + TableOfContentsProvider: page.NopPage, + } +} + +func (p *pageForRenderHooks) page() page.Page { + return p.PageWithoutContent.(page.Page) +} diff --git a/hugolib/site.go b/hugolib/site.go index 67ddff4d901..a1313d7e3b1 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -28,6 +28,10 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/identity" + + "github.com/gohugoio/hugo/markup/converter/hooks" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/markup/converter" @@ -801,7 +805,6 @@ func (s *Site) multilingual() *Multilingual { type whatChanged struct { source bool - other bool files map[string]bool } @@ -888,10 +891,11 @@ func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event { // It returns whetever the content source was changed. // TODO(bep) clean up/rewrite this method. func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error { - events = s.filterFileEvents(events) events = s.translateFileEvents(events) + changeIdentities := make(identity.Identities) + s.Log.DEBUG.Printf("Rebuild for events %q", events) h := s.h @@ -902,11 +906,12 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro sourceChanged = []fsnotify.Event{} sourceReallyChanged = []fsnotify.Event{} contentFilesChanged []string - tmplChanged = []fsnotify.Event{} - dataChanged = []fsnotify.Event{} - i18nChanged = []fsnotify.Event{} - shortcodesChanged = make(map[string]bool) - sourceFilesChanged = make(map[string]bool) + + tmplChanged bool + dataChanged bool + i18nChanged bool + + sourceFilesChanged = make(map[string]bool) // prevent spamming the log on changes logger = helpers.NewDistinctFeedbackLogger() @@ -915,37 +920,34 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro var cachePartitions []string for _, ev := range events { - if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { - cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) - } + id, found := s.eventToIdentity(ev) + if found { + changeIdentities[id] = id + + if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { + cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) + } + + switch id.Type { + case files.ComponentFolderContent: + logger.Println("Source changed", ev) + sourceChanged = append(sourceChanged, ev) + case files.ComponentFolderLayouts: + logger.Println("Template changed", ev) + tmplChanged = true + case files.ComponentFolderData: + logger.Println("Data changed", ev) + dataChanged = true + case files.ComponentFolderI18n: + logger.Println("i18n changed", ev) + i18nChanged = true - if s.isContentDirEvent(ev) { - logger.Println("Source changed", ev) - sourceChanged = append(sourceChanged, ev) - } - if s.isLayoutDirEvent(ev) { - logger.Println("Template changed", ev) - tmplChanged = append(tmplChanged, ev) - - if strings.Contains(ev.Name, "shortcodes") { - shortcode := filepath.Base(ev.Name) - shortcode = strings.TrimSuffix(shortcode, filepath.Ext(shortcode)) - shortcodesChanged[shortcode] = true } - } - if s.isDataDirEvent(ev) { - logger.Println("Data changed", ev) - dataChanged = append(dataChanged, ev) - } - if s.isI18nEvent(ev) { - logger.Println("i18n changed", ev) - i18nChanged = append(dataChanged, ev) } } changed := &whatChanged{ - source: len(sourceChanged) > 0 || len(shortcodesChanged) > 0, - other: len(tmplChanged) > 0 || len(i18nChanged) > 0 || len(dataChanged) > 0, + source: len(sourceChanged) > 0, files: sourceFilesChanged, } @@ -960,7 +962,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) } - if len(tmplChanged) > 0 || len(i18nChanged) > 0 { + if tmplChanged || i18nChanged { sites := s.h.Sites first := sites[0] @@ -989,7 +991,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro } } - if len(dataChanged) > 0 { + if dataChanged { s.h.init.data.Reset() } @@ -1018,18 +1020,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro sourceFilesChanged[ev.Name] = true } - for shortcode := range shortcodesChanged { - // There are certain scenarios that, when a shortcode changes, - // it isn't sufficient to just rerender the already parsed shortcode. - // One example is if the user adds a new shortcode to the content file first, - // and then creates the shortcode on the file system. - // To handle these scenarios, we must do a full reprocessing of the - // pages that keeps a reference to the changed shortcode. - pagesWithShortcode := h.findPagesByShortcode(shortcode) - for _, p := range pagesWithShortcode { - contentFilesChanged = append(contentFilesChanged, p.File().Filename()) - } - } + h.resetPageStateFromEvents(changeIdentities) if len(sourceReallyChanged) > 0 || len(contentFilesChanged) > 0 { var filenamesChanged []string @@ -1218,20 +1209,14 @@ func (s *Site) initializeSiteInfo() error { return nil } -func (s *Site) isI18nEvent(e fsnotify.Event) bool { - return s.BaseFs.SourceFilesystems.IsI18n(e.Name) -} - -func (s *Site) isDataDirEvent(e fsnotify.Event) bool { - return s.BaseFs.SourceFilesystems.IsData(e.Name) -} - -func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool { - return s.BaseFs.SourceFilesystems.IsLayout(e.Name) -} +func (s *Site) eventToIdentity(e fsnotify.Event) (identity.PathIdentity, bool) { + for _, fs := range s.BaseFs.SourceFilesystems.FileSystems() { + if p := fs.Path(e.Name); p != "" { + return identity.NewPathIdentity(fs.Name, p), true + } + } -func (s *Site) isContentDirEvent(e fsnotify.Event) bool { - return s.BaseFs.IsContent(e.Name) + return identity.PathIdentity{}, false } func (s *Site) readAndProcessContent(filenames ...string) error { @@ -1562,6 +1547,26 @@ var infoOnMissingLayout = map[string]bool{ "404": true, } +type contentLinkRenderer struct { + templateHandler tpl.TemplateHandler + identity.Provider + templ tpl.Template +} + +func (r contentLinkRenderer) Render(w io.Writer, ctx hooks.LinkContext) error { + return r.templateHandler.Execute(r.templ, w, ctx) +} + +func (s *Site) lookupTemplate(layouts ...string) (tpl.Template, bool) { + for _, l := range layouts { + if templ, found := s.Tmpl.Lookup(l); found { + return templ, true + } + } + + return nil, false +} + func (s *Site) renderForLayouts(name, outputFormat string, d interface{}, w io.Writer, layouts ...string) (err error) { templ := s.findFirstTemplate(layouts...) if templ == nil { diff --git a/hugolib/site_benchmark_new_test.go b/hugolib/site_benchmark_new_test.go index 646124b09ca..13302300ee9 100644 --- a/hugolib/site_benchmark_new_test.go +++ b/hugolib/site_benchmark_new_test.go @@ -127,6 +127,36 @@ title = "What is Markdown" baseURL = "https://example.com" `) + + data, err := ioutil.ReadFile(filepath.FromSlash("testdata/what-is-markdown.md")) + sb.Assert(err, qt.IsNil) + datastr := string(data) + getContent := func(i int) string { + return fmt.Sprintf(`--- +title: "Page %d" +--- + +`, i) + datastr + + } + for i := 1; i <= 100; i++ { + sb.WithContent(fmt.Sprintf("content/page%d.md", i), getContent(i)) + } + + return sb + }, + func(s *sitesBuilder) { + s.Assert(s.CheckExists("public/page8/index.html"), qt.Equals, true) + }, + }, + {"Markdown with custom link handler", func(b testing.TB) *sitesBuilder { + sb := newTestSitesBuilder(b).WithConfigFile("toml", ` +title = "What is Markdown" +baseURL = "https://example.com" + +`) + + sb.WithTemplatesAdded("_default/_markup/render-link.html", `CUSTOM LINK`) data, err := ioutil.ReadFile(filepath.FromSlash("testdata/what-is-markdown.md")) sb.Assert(err, qt.IsNil) datastr := string(data) diff --git a/hugolib/template_test.go b/hugolib/template_test.go index 71b4b46c0bf..120f399964a 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -18,6 +18,12 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/identity" + + "github.com/gohugoio/hugo/tpl" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" @@ -320,6 +326,7 @@ Partial cached1: {{ partialCached "p1" "input1" $key1 }} Partial cached2: {{ partialCached "p1" "input2" $key1 }} Partial cached3: {{ partialCached "p1" "input3" $key2 }} `, + "partials/p1.html", `partial: {{ . }}`, ) @@ -331,3 +338,85 @@ Partial cached3: {{ partialCached "p1" "input3" $key2 }} Partial cached3: partial: input3 `) } + +func TestTemplateDependencies(t *testing.T) { + b := newTestSitesBuilder(t).Running() + + b.WithTemplates("index.html", ` +{{ $p := site.GetPage "p1" }} +{{ partial "p1.html" $p }} +{{ partialCached "p2.html" "foo" }} +{{ partials.Include "p3.html" "data" }} +{{ partials.IncludeCached "p4.html" "foo" }} +{{ $p := partial "p5" }} +{{ partial "sub/p6.html" }} +{{ partial "P7.html" }} +{{ template "_default/foo.html" }} +Partial nested: {{ partial "p10" }} + +`, + "partials/p1.html", `ps: {{ .Render "li" }}`, + "partials/p2.html", `p2`, + "partials/p3.html", `p3`, + "partials/p4.html", `p4`, + "partials/p5.html", `p5`, + "partials/sub/p6.html", `p6`, + "partials/P7.html", `p7`, + "partials/p8.html", `p8 {{ partial "p9.html" }}`, + "partials/p9.html", `p9`, + "partials/p10.html", `p10 {{ partial "p11.html" }}`, + "partials/p11.html", `p11`, + "_default/foo.html", `foo`, + "_default/li.html", `li {{ partial "p8.html" }}`, + ) + + b.WithContent("p1.md", `--- +title: P1 +--- + + +`) + + b.Build(BuildCfg{}) + + s := b.H.Sites[0] + + templ, found := s.lookupTemplate("index.html") + b.Assert(found, qt.Equals, true) + + idset := make(map[identity.Identity]bool) + collectIdentities(idset, templ.(tpl.TemplateInfoProvider).TemplateInfo()) + b.Assert(idset, qt.HasLen, 10) + +} + +func collectIdentities(set map[identity.Identity]bool, provider identity.Provider) { + if ids, ok := provider.(identity.IdentitiesProvider); ok { + for _, id := range ids.GetIdentities() { + collectIdentities(set, id) + } + } else { + set[provider.GetIdentity()] = true + } +} + +func printRecursiveIdentities(level int, id identity.Provider) { + if level == 0 { + fmt.Println(id.GetIdentity(), "===>") + } + if ids, ok := id.(identity.IdentitiesProvider); ok { + level++ + for _, id := range ids.GetIdentities() { + printRecursiveIdentities(level, id) + } + } else { + ident(level) + fmt.Println("ID", id) + } +} + +func ident(n int) { + for i := 0; i < n; i++ { + fmt.Print(" ") + } +} diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index ea1ee967499..80aafe052ef 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -389,8 +389,9 @@ func (s *sitesBuilder) EditFiles(filenameContent ...string) *sitesBuilder { var changedFiles []string for i := 0; i < len(filenameContent); i += 2 { filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] - changedFiles = append(changedFiles, filename) - writeSource(s.T, s.Fs, s.absFilename(filename), content) + absFilename := s.absFilename(filename) + changedFiles = append(changedFiles, absFilename) + writeSource(s.T, s.Fs, absFilename, content) } s.changedFiles = changedFiles @@ -963,10 +964,6 @@ func isCI() bool { return os.Getenv("CI") != "" } -func isGo111() bool { - return strings.Contains(runtime.Version(), "1.11") -} - // See https://github.com/golang/go/issues/19280 // Not in use. var parallelEnabled = true diff --git a/identity/identity.go b/identity/identity.go new file mode 100644 index 00000000000..bf03189cc9e --- /dev/null +++ b/identity/identity.go @@ -0,0 +1,131 @@ +package identity + +import ( + "path/filepath" + "strings" + "sync" +) + +// NewIdentityManager creates a new Manager starting at id. +func NewIdentityManager(id Provider) Manager { + return &identityManager{ + Provider: id, + ids: Identities{id.GetIdentity(): id}, + } +} + +// NewPathIdentity creates a new Identity with the two identifiers +// type and path. +func NewPathIdentity(typ, pat string) PathIdentity { + pat = strings.ToLower(strings.TrimPrefix(filepath.ToSlash(pat), "/")) + return PathIdentity{Type: typ, Path: pat} +} + +// Identities stores identity providers. +type Identities map[Identity]Provider + +func (ids Identities) search(id Identity) Provider { + if v, found := ids[id]; found { + return v + } + for _, v := range ids { + switch t := v.(type) { + case IdentitiesProvider: + if nested := t.GetIdentities().search(id); nested != nil { + return nested + } + } + } + return nil +} + +// IdentitiesProvider provides all Identities. +type IdentitiesProvider interface { + GetIdentities() Identities +} + +// Identity represents an thing that can provide an identify. This can be +// any Go type, but the Identity returned by GetIdentify must be hashable. +type Identity interface { + Provider + Name() string +} + +// Manager manages identities, and is itself a Provider of Identity. +type Manager interface { + IdentitiesProvider + Provider + Add(ids ...Provider) + Search(id Identity) Provider + Reset() +} + +// A PathIdentity is a common identity identified by a type and a path, e.g. "layouts" and "_default/single.html". +type PathIdentity struct { + Type string + Path string +} + +// GetIdentity returns itself. +func (id PathIdentity) GetIdentity() Identity { + return id +} + +// Name returns the Path. +func (id PathIdentity) Name() string { + return id.Path +} + +// A KeyValueIdentity a general purpose identity. +type KeyValueIdentity struct { + Key string + Value string +} + +// GetIdentity returns itself. +func (id KeyValueIdentity) GetIdentity() Identity { + return id +} + +// Name returns the Key. +func (id KeyValueIdentity) Name() string { + return id.Key +} + +// Provider provides the hashable Identity. +type Provider interface { + GetIdentity() Identity +} + +type identityManager struct { + sync.Mutex + Provider + ids Identities +} + +func (im *identityManager) Add(ids ...Provider) { + im.Lock() + for _, id := range ids { + im.ids[id.GetIdentity()] = id + } + im.Unlock() +} + +func (im *identityManager) Reset() { + im.Lock() + id := im.GetIdentity() + im.ids = Identities{id.GetIdentity(): id} + im.Unlock() +} + +func (im *identityManager) GetIdentities() Identities { + im.Lock() + defer im.Unlock() + return im.ids +} + +func (im *identityManager) Search(id Identity) Provider { + im.Lock() + defer im.Unlock() + return im.ids.search(id.GetIdentity()) +} diff --git a/identity/identity_test.go b/identity/identity_test.go new file mode 100644 index 00000000000..78e7a3b5e15 --- /dev/null +++ b/identity/identity_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package identity + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestIdentityManager(t *testing.T) { + c := qt.New(t) + + id1 := testIdentity{name: "id1"} + im := NewIdentityManager(id1) + + c.Assert(im.Search(id1).GetIdentity(), qt.Equals, id1) + c.Assert(im.Search(testIdentity{name: "notfound"}), qt.Equals, nil) +} + +type testIdentity struct { + name string +} + +func (id testIdentity) GetIdentity() Identity { + return id +} + +func (id testIdentity) Name() string { + return id.name +} diff --git a/markup/asciidoc/convert.go b/markup/asciidoc/convert.go index 65fdde0f564..a72aac39198 100644 --- a/markup/asciidoc/convert.go +++ b/markup/asciidoc/convert.go @@ -18,6 +18,7 @@ package asciidoc import ( "os/exec" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" "github.com/gohugoio/hugo/markup/converter" @@ -47,6 +48,10 @@ func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Resu return converter.Bytes(a.getAsciidocContent(ctx.Src, a.ctx)), nil } +func (c *asciidocConverter) Supports(feature identity.Identity) bool { + return false +} + // getAsciidocContent calls asciidoctor or asciidoc as an external helper // to convert AsciiDoc content to HTML. func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte { diff --git a/markup/blackfriday/convert.go b/markup/blackfriday/convert.go index 350defcb63c..3df23c7ae74 100644 --- a/markup/blackfriday/convert.go +++ b/markup/blackfriday/convert.go @@ -15,6 +15,7 @@ package blackfriday import ( + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config" "github.com/gohugoio/hugo/markup/converter" "github.com/russross/blackfriday" @@ -72,6 +73,10 @@ func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.R return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil } +func (c *blackfridayConverter) Supports(feature identity.Identity) bool { + return false +} + func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer { flags := getFlags(renderTOC, c.bf) diff --git a/markup/converter/converter.go b/markup/converter/converter.go index a1141f65ccc..a4585bd0380 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -16,6 +16,8 @@ package converter import ( "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/spf13/afero" @@ -67,6 +69,7 @@ func (n newConverter) Name() string { // another format, e.g. Markdown to HTML. type Converter interface { Convert(ctx RenderContext) (Result, error) + Supports(feature identity.Identity) bool } // Result represents the minimum returned from Convert. @@ -94,6 +97,7 @@ func (b Bytes) Bytes() []byte { // DocumentContext holds contextual information about the document to convert. type DocumentContext struct { + Document interface{} // May be nil. Usually a page.Page DocumentID string DocumentName string ConfigOverrides map[string]interface{} @@ -101,6 +105,11 @@ type DocumentContext struct { // RenderContext holds contextual information about the content to render. type RenderContext struct { - Src []byte - RenderTOC bool + Src []byte + RenderTOC bool + RenderHooks *hooks.Render } + +var ( + FeatureRenderHooks = identity.NewPathIdentity("markup", "renderingHooks") +) diff --git a/markup/converter/hooks/hooks.go b/markup/converter/hooks/hooks.go new file mode 100644 index 00000000000..3be1fd1c3c6 --- /dev/null +++ b/markup/converter/hooks/hooks.go @@ -0,0 +1,58 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hooks + +import ( + "io" + + "github.com/gohugoio/hugo/identity" +) + +type LinkContext interface { + Page() interface{} + Destination() string + Title() string + Text() string + Resolved() bool // TODO1 consider +} + +type Render struct { + LinkRenderer LinkRenderer + ImageRenderer LinkRenderer +} + +func (r *Render) Eq(other interface{}) bool { + ro, ok := other.(*Render) + if !ok { + return false + } + if r == nil || ro == nil { + return r == nil + } + + if r.ImageRenderer.GetIdentity() != ro.ImageRenderer.GetIdentity() { + return false + } + + if r.LinkRenderer.GetIdentity() != ro.LinkRenderer.GetIdentity() { + return false + } + + return true +} + +type LinkRenderer interface { + Render(w io.Writer, ctx LinkContext) error + identity.Provider +} diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go index 15b0f0d77c8..8c7414af1a4 100644 --- a/markup/goldmark/convert.go +++ b/markup/goldmark/convert.go @@ -15,21 +15,22 @@ package goldmark import ( + "bufio" "bytes" "fmt" "path/filepath" "runtime/debug" + "github.com/gohugoio/hugo/identity" + "github.com/pkg/errors" "github.com/spf13/afero" "github.com/gohugoio/hugo/hugofs" - "github.com/alecthomas/chroma/styles" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/highlight" - "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/yuin/goldmark" hl "github.com/yuin/goldmark-highlighting" @@ -48,7 +49,7 @@ type provide struct { } func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) { - md := newMarkdown(cfg.MarkupConfig) + md := newMarkdown(cfg) return converter.NewProvider("goldmark", func(ctx converter.DocumentContext) (converter.Converter, error) { return &goldmarkConverter{ ctx: ctx, @@ -64,11 +65,13 @@ type goldmarkConverter struct { cfg converter.ProviderConfig } -func newMarkdown(mcfg markup_config.Config) goldmark.Markdown { - cfg := mcfg.Goldmark +func newMarkdown(pcfg converter.ProviderConfig) goldmark.Markdown { + mcfg := pcfg.MarkupConfig + cfg := pcfg.MarkupConfig.Goldmark var ( extensions = []goldmark.Extender{ + newLinks(), newTocExtension(), } rendererOptions []renderer.Option @@ -143,15 +146,53 @@ func newMarkdown(mcfg markup_config.Config) goldmark.Markdown { } +var _ identity.IdentitiesProvider = (*converterResult)(nil) + type converterResult struct { converter.Result toc tableofcontents.Root + ids identity.Identities } func (c converterResult) TableOfContents() tableofcontents.Root { return c.toc } +func (c converterResult) GetIdentities() identity.Identities { + return c.ids +} + +type renderContext struct { + util.BufWriter + renderContextData +} + +type renderContextData interface { + RenderContext() converter.RenderContext + DocumentContext() converter.DocumentContext + AddIdentity(id identity.Identity) +} + +type renderContextDataHolder struct { + rctx converter.RenderContext + dctx converter.DocumentContext + ids identity.Manager +} + +func (ctx *renderContextDataHolder) RenderContext() converter.RenderContext { + return ctx.rctx +} + +func (ctx *renderContextDataHolder) DocumentContext() converter.DocumentContext { + return ctx.dctx +} + +func (ctx *renderContextDataHolder) AddIdentity(id identity.Identity) { + ctx.ids.Add(id) +} + +var converterIdentity = identity.KeyValueIdentity{Key: "goldmark", Value: "converter"} + func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result converter.Result, err error) { defer func() { if r := recover(); r != nil { @@ -166,9 +207,7 @@ func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result convert buf := &bytes.Buffer{} result = buf - pctx := parser.NewContext() - pctx.Set(tocEnableKey, ctx.RenderTOC) - + pctx := newParserContext(ctx) reader := text.NewReader(ctx.Src) doc := c.md.Parser().Parse( @@ -176,27 +215,58 @@ func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result convert parser.WithContext(pctx), ) - if err := c.md.Renderer().Render(buf, ctx.Src, doc); err != nil { + rcx := &renderContextDataHolder{ + rctx: ctx, + dctx: c.ctx, + ids: identity.NewIdentityManager(converterIdentity), + } + + w := renderContext{ + BufWriter: bufio.NewWriter(buf), + renderContextData: rcx, + } + + if err := c.md.Renderer().Render(w, ctx.Src, doc); err != nil { return nil, err } - if toc, ok := pctx.Get(tocResultKey).(tableofcontents.Root); ok { - return converterResult{ - Result: buf, - toc: toc, - }, nil + return converterResult{ + Result: buf, + ids: rcx.ids.GetIdentities(), + toc: pctx.TableOfContents(), + }, nil + +} + +var featureSet = map[identity.Identity]bool{ + converter.FeatureRenderHooks: true, +} + +func (c *goldmarkConverter) Supports(feature identity.Identity) bool { + return featureSet[feature.GetIdentity()] +} + +func newParserContext(rctx converter.RenderContext) *parserContext { + ctx := parser.NewContext() + ctx.Set(tocEnableKey, rctx.RenderTOC) + return &parserContext{ + Context: ctx, } +} - return buf, nil +type parserContext struct { + parser.Context } -func newHighlighting(cfg highlight.Config) goldmark.Extender { - style := styles.Get(cfg.Style) - if style == nil { - style = styles.Fallback +func (p *parserContext) TableOfContents() tableofcontents.Root { + if v := p.Get(tocResultKey); v != nil { + return v.(tableofcontents.Root) } + return tableofcontents.Root{} +} - e := hl.NewHighlighting( +func newHighlighting(cfg highlight.Config) goldmark.Extender { + return hl.NewHighlighting( hl.WithStyle(cfg.Style), hl.WithGuessLanguage(cfg.GuessSyntax), hl.WithCodeBlockOptions(highlight.GetCodeBlockOptions()), @@ -230,6 +300,4 @@ func newHighlighting(cfg highlight.Config) goldmark.Extender { }), ) - - return e } diff --git a/markup/goldmark/convert_test.go b/markup/goldmark/convert_test.go index b6816d2e54a..2a97276064b 100644 --- a/markup/goldmark/convert_test.go +++ b/markup/goldmark/convert_test.go @@ -38,6 +38,9 @@ func TestConvert(t *testing.T) { https://github.com/gohugoio/hugo/issues/6528 [Live Demo here!](https://docuapi.netlify.com/) +[I'm an inline-style link with title](https://www.google.com "Google's Homepage") + + ## Code Fences §§§bash @@ -98,6 +101,7 @@ description mconf := markup_config.Default mconf.Highlight.NoClasses = false + mconf.Goldmark.Renderer.Unsafe = true p, err := Provider.New( converter.ProviderConfig{ @@ -106,15 +110,15 @@ description }, ) c.Assert(err, qt.IsNil) - conv, err := p.New(converter.DocumentContext{}) + conv, err := p.New(converter.DocumentContext{DocumentID: "thedoc"}) c.Assert(err, qt.IsNil) - b, err := conv.Convert(converter.RenderContext{Src: []byte(content)}) + b, err := conv.Convert(converter.RenderContext{RenderTOC: true, Src: []byte(content)}) c.Assert(err, qt.IsNil) got := string(b.Bytes()) // Links - c.Assert(got, qt.Contains, `Live Demo here!`) + // c.Assert(got, qt.Contains, `Live Demo here!`) // Header IDs c.Assert(got, qt.Contains, `

Custom ID

`, qt.Commentf(got)) @@ -137,6 +141,11 @@ description c.Assert(got, qt.Contains, `
`) c.Assert(got, qt.Contains, `
date
`) + toc, ok := b.(converter.TableOfContentsProvider) + c.Assert(ok, qt.Equals, true) + tocHTML := toc.TableOfContents().ToHTML(1, 2, false) + c.Assert(tocHTML, qt.Contains, "TableOfContents") + } func TestCodeFence(t *testing.T) { diff --git a/markup/goldmark/render_link.go b/markup/goldmark/render_link.go new file mode 100644 index 00000000000..17ba5badacc --- /dev/null +++ b/markup/goldmark/render_link.go @@ -0,0 +1,208 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goldmark + +import ( + "github.com/gohugoio/hugo/markup/converter/hooks" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/util" +) + +var _ renderer.SetOptioner = (*linkRenderer)(nil) + +func newLinkRenderer() renderer.NodeRenderer { + r := &linkRenderer{ + Config: html.Config{ + Writer: html.DefaultWriter, + }, + } + return r +} + +func newLinks() goldmark.Extender { + return &links{} +} + +type linkContext struct { + page interface{} + destination string + title string + text string +} + +func (ctx linkContext) Destination() string { + return ctx.destination +} + +func (ctx linkContext) Resolved() bool { + return false +} + +func (ctx linkContext) Page() interface{} { + return ctx.page +} + +func (ctx linkContext) Text() string { + return ctx.text +} + +func (ctx linkContext) Title() string { + return ctx.title +} + +type linkRenderer struct { + html.Config +} + +func (r *linkRenderer) SetOption(name renderer.OptionName, value interface{}) { + r.Config.SetOption(name, value) +} + +// RegisterFuncs implements NodeRenderer.RegisterFuncs. +func (r *linkRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindLink, r.renderLink) + reg.Register(ast.KindImage, r.renderImage) +} + +// Fall back to the default Goldmark render funcs. Method below borrowed from: +// https://github.com/yuin/goldmark/blob/b611cd333a492416b56aa8d94b04a67bf0096ab2/renderer/html/html.go#L404 +func (r *linkRenderer) renderDefaultImage(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + n := node.(*ast.Image) + _, _ = w.WriteString("`)
+	_, _ = w.Write(n.Text(source))
+	_ = w.WriteByte('") + } else { + _, _ = w.WriteString(">") + } + return ast.WalkSkipChildren, nil +} + +// Fall back to the default Goldmark render funcs. Method below borrowed from: +// https://github.com/yuin/goldmark/blob/b611cd333a492416b56aa8d94b04a67bf0096ab2/renderer/html/html.go#L404 +func (r *linkRenderer) renderDefaultLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Link) + if entering { + _, _ = w.WriteString("') + } else { + _, _ = w.WriteString("") + } + return ast.WalkContinue, nil +} + +func (r *linkRenderer) renderImage(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Image) + var h *hooks.Render + + ctx, ok := w.(renderContextData) + if ok { + h = ctx.RenderContext().RenderHooks + ok = h != nil && h.ImageRenderer != nil + } + + if !ok { + return r.renderDefaultImage(w, source, node, entering) + } + + if !entering { + return ast.WalkContinue, nil + } + + err := h.ImageRenderer.Render( + w, + linkContext{ + page: ctx.DocumentContext().Document, + destination: string(n.Destination), + title: string(n.Title), + text: string(n.Text(source)), + }, + ) + + ctx.AddIdentity(h.ImageRenderer.GetIdentity()) + + return ast.WalkSkipChildren, err + +} + +func (r *linkRenderer) renderLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Link) + var h *hooks.Render + + ctx, ok := w.(renderContextData) + if ok { + h = ctx.RenderContext().RenderHooks + ok = h != nil && h.LinkRenderer != nil + } + + if !ok { + return r.renderDefaultLink(w, source, node, entering) + } + + if !entering { + return ast.WalkContinue, nil + } + + err := h.LinkRenderer.Render( + w, + linkContext{ + page: ctx.DocumentContext().Document, + destination: string(n.Destination), + title: string(n.Title), + text: string(n.Text(source)), + }, + ) + + ctx.AddIdentity(h.LinkRenderer.GetIdentity()) + + // Do not render the inner text. + return ast.WalkSkipChildren, err + +} + +type links struct { +} + +// Extend implements goldmark.Extender. +func (e *links) Extend(m goldmark.Markdown) { + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(newLinkRenderer(), 100), + )) +} diff --git a/markup/mmark/convert.go b/markup/mmark/convert.go index 07b2a6f81e5..0682ad276c6 100644 --- a/markup/mmark/convert.go +++ b/markup/mmark/convert.go @@ -15,6 +15,7 @@ package mmark import ( + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config" "github.com/gohugoio/hugo/markup/converter" "github.com/miekg/mmark" @@ -65,6 +66,10 @@ func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result, return mmark.Parse(ctx.Src, r, c.extensions), nil } +func (c *mmarkConverter) Supports(feature identity.Identity) bool { + return false +} + func getHTMLRenderer( ctx converter.DocumentContext, cfg blackfriday_config.Config, diff --git a/markup/org/convert.go b/markup/org/convert.go index 4d6e5e2fa0f..2b1fbb73c3a 100644 --- a/markup/org/convert.go +++ b/markup/org/convert.go @@ -17,6 +17,8 @@ package org import ( "bytes" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/converter" "github.com/niklasfasching/go-org/org" "github.com/spf13/afero" @@ -66,3 +68,7 @@ func (c *orgConverter) Convert(ctx converter.RenderContext) (converter.Result, e } return converter.Bytes([]byte(html)), nil } + +func (c *orgConverter) Supports(feature identity.Identity) bool { + return false +} diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go index d538d4a5265..d6d5ab18c8c 100644 --- a/markup/pandoc/convert.go +++ b/markup/pandoc/convert.go @@ -17,6 +17,7 @@ package pandoc import ( "os/exec" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" "github.com/gohugoio/hugo/markup/converter" @@ -47,6 +48,10 @@ func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil } +func (c *pandocConverter) Supports(feature identity.Identity) bool { + return false +} + // getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte { logger := c.cfg.Logger diff --git a/markup/rst/convert.go b/markup/rst/convert.go index 040b40d792d..64cc8b5114f 100644 --- a/markup/rst/convert.go +++ b/markup/rst/convert.go @@ -19,6 +19,7 @@ import ( "os/exec" "runtime" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" "github.com/gohugoio/hugo/markup/converter" @@ -48,6 +49,10 @@ func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, e return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil } +func (c *rstConverter) Supports(feature identity.Identity) bool { + return false +} + // getRstContent calls the Python script rst2html as an external helper // to convert reStructuredText content to HTML. func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte { diff --git a/output/layout.go b/output/layout.go index 055d742b15f..7d935d0ba89 100644 --- a/output/layout.go +++ b/output/layout.go @@ -37,6 +37,12 @@ type LayoutDescriptor struct { Layout string // LayoutOverride indicates what we should only look for the above layout. LayoutOverride bool + + RenderingHook bool +} + +func (d LayoutDescriptor) isList() bool { + return !d.RenderingHook && d.Kind != "page" } // LayoutHandler calculates the layout template to use to render a given output type. @@ -89,7 +95,7 @@ type layoutBuilder struct { func (l *layoutBuilder) addLayoutVariations(vars ...string) { for _, layoutVar := range vars { - if l.d.LayoutOverride && layoutVar != l.d.Layout { + if !l.d.RenderingHook && l.d.LayoutOverride && layoutVar != l.d.Layout { continue } l.layoutVariations = append(l.layoutVariations, layoutVar) @@ -99,6 +105,9 @@ func (l *layoutBuilder) addLayoutVariations(vars ...string) { func (l *layoutBuilder) addTypeVariations(vars ...string) { for _, typeVar := range vars { if !reservedSections[typeVar] { + if l.d.RenderingHook { + typeVar = typeVar + renderingHookRoot + } l.typeVariations = append(l.typeVariations, typeVar) } } @@ -115,16 +124,25 @@ func (l *layoutBuilder) addKind() { l.addTypeVariations(l.d.Kind) } +const renderingHookRoot = "/_markup" + func resolvePageTemplate(d LayoutDescriptor, f Format) []string { b := &layoutBuilder{d: d, f: f} - if d.Layout != "" { - b.addLayoutVariations(d.Layout) - } - - if d.Type != "" { - b.addTypeVariations(d.Type) + if d.RenderingHook { + if d.Type != "" { + b.addTypeVariations(d.Type) + } + b.addLayoutVariations(d.Kind) + b.addSectionType() + } else { + if d.Layout != "" { + b.addLayoutVariations(d.Layout) + } + if d.Type != "" { + b.addTypeVariations(d.Type) + } } switch d.Kind { @@ -159,7 +177,7 @@ func resolvePageTemplate(d LayoutDescriptor, f Format) []string { } isRSS := f.Name == RSSFormat.Name - if isRSS { + if !d.RenderingHook && isRSS { // The historic and common rss.xml case b.addLayoutVariations("") } @@ -167,14 +185,14 @@ func resolvePageTemplate(d LayoutDescriptor, f Format) []string { // All have _default in their lookup path b.addTypeVariations("_default") - if d.Kind != "page" { + if d.isList() { // Add the common list type b.addLayoutVariations("list") } layouts := b.resolveVariations() - if isRSS { + if !d.RenderingHook && isRSS { layouts = append(layouts, "_internal/_default/rss.xml") } diff --git a/output/layout_test.go b/output/layout_test.go index c6267b27434..9e4f89098c3 100644 --- a/output/layout_test.go +++ b/output/layout_test.go @@ -111,6 +111,8 @@ func TestLayout(t *testing.T) { []string{"section/shortcodes.amp.html"}, 12}, {"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, "", ampType, []string{"section/partials.amp.html"}, 12}, + {"Content hook", LayoutDescriptor{Kind: "render-link", RenderingHook: true, Section: "blog"}, "", ampType, + []string{"blog/_markup/render-link.amp.html", "blog/_markup/render-link.html", "_default/_markup/render-link.amp.html", "_default/_markup/render-link.html"}, 4}, } { c.Run(this.name, func(c *qt.C) { l := NewLayoutHandler() diff --git a/public/categories/index.xml b/public/categories/index.xml new file mode 100644 index 00000000000..ae8c7f8f749 --- /dev/null +++ b/public/categories/index.xml @@ -0,0 +1,13 @@ + + + + Categories on + /categories/ + Recent content in Categories on + Hugo -- gohugo.io + + + + + + \ No newline at end of file diff --git a/public/index.xml b/public/index.xml new file mode 100644 index 00000000000..b70aeed6e08 --- /dev/null +++ b/public/index.xml @@ -0,0 +1,13 @@ + + + + + / + Recent content on + Hugo -- gohugo.io + + + + + + \ No newline at end of file diff --git a/public/sitemap.xml b/public/sitemap.xml new file mode 100644 index 00000000000..95cee9f7cdc --- /dev/null +++ b/public/sitemap.xml @@ -0,0 +1,17 @@ + + + + + / + + + + /categories/ + + + + /tags/ + + + \ No newline at end of file diff --git a/public/tags/index.xml b/public/tags/index.xml new file mode 100644 index 00000000000..43dbc43baa7 --- /dev/null +++ b/public/tags/index.xml @@ -0,0 +1,13 @@ + + + + Tags on + /tags/ + Recent content in Tags on + Hugo -- gohugo.io + + + + + + \ No newline at end of file diff --git a/resources/page/page.go b/resources/page/page.go index 3b43b0af3f1..2ae23792e29 100644 --- a/resources/page/page.go +++ b/resources/page/page.go @@ -201,9 +201,10 @@ type PageMetaProvider interface { Weight() int } -// PageRenderProvider provides a way for a Page to render itself. +// PageRenderProvider provides a way for a Page to render content. type PageRenderProvider interface { - Render(layout ...string) template.HTML + Render(layout ...string) (template.HTML, error) + RenderString(s interface{}) (template.HTML, error) // TODO1 inline option? } // PageWithoutContent is the Page without any of the content methods. diff --git a/resources/page/page_nop.go b/resources/page/page_nop.go index 09ac136fc2b..43a7b7b4689 100644 --- a/resources/page/page_nop.go +++ b/resources/page/page_nop.go @@ -371,8 +371,12 @@ func (p *nopPage) RelRef(argsm map[string]interface{}) (string, error) { return "", nil } -func (p *nopPage) Render(layout ...string) template.HTML { - return "" +func (p *nopPage) Render(layout ...string) (template.HTML, error) { + return "", nil +} + +func (p *nopPage) RenderString(s interface{}) (template.HTML, error) { + return "", nil } func (p *nopPage) ResourceType() string { diff --git a/resources/page/testhelpers_test.go b/resources/page/testhelpers_test.go index cc6a74f06de..746a881f9d2 100644 --- a/resources/page/testhelpers_test.go +++ b/resources/page/testhelpers_test.go @@ -446,7 +446,11 @@ func (p *testPage) RelRefFrom(argsm map[string]interface{}, source interface{}) return "", nil } -func (p *testPage) Render(layout ...string) template.HTML { +func (p *testPage) Render(layout ...string) (template.HTML, error) { + panic("not implemented") +} + +func (p *testPage) RenderString(s interface{}) (template.HTML, error) { panic("not implemented") } diff --git a/scripts/fork_go_templates/main.go b/scripts/fork_go_templates/main.go index 1cae78a4368..127d7ce0374 100644 --- a/scripts/fork_go_templates/main.go +++ b/scripts/fork_go_templates/main.go @@ -59,6 +59,7 @@ var ( "type state struct", "type stateOld struct", "func (s *state) evalFunction", "func (s *state) evalFunctionOld", "func (s *state) evalField(", "func (s *state) evalFieldOld(", + "func (s *state) evalCall(", "func (s *state) evalCallOld(", ) htmlTemplateReplacers = strings.NewReplacer( diff --git a/tpl/internal/go_templates/texttemplate/exec.go b/tpl/internal/go_templates/texttemplate/exec.go index db64edcb27e..078bcf643d6 100644 --- a/tpl/internal/go_templates/texttemplate/exec.go +++ b/tpl/internal/go_templates/texttemplate/exec.go @@ -658,7 +658,7 @@ var ( // evalCall executes a function or method call. If it's a method, fun already has the receiver bound, so // it looks just like a function call. The arg list, if non-nil, includes (in the manner of the shell), arg[0] // as the function itself. -func (s *state) evalCall(dot, fun reflect.Value, node parse.Node, name string, args []parse.Node, final reflect.Value) reflect.Value { +func (s *state) evalCallOld(dot, fun reflect.Value, node parse.Node, name string, args []parse.Node, final reflect.Value) reflect.Value { if args != nil { args = args[1:] // Zeroth arg is function name/node; not passed to function. } diff --git a/tpl/internal/go_templates/texttemplate/hugo_template.go b/tpl/internal/go_templates/texttemplate/hugo_template.go index be8a5558f50..ba1b9ec86cb 100644 --- a/tpl/internal/go_templates/texttemplate/hugo_template.go +++ b/tpl/internal/go_templates/texttemplate/hugo_template.go @@ -34,8 +34,9 @@ type Preparer interface { // ExecHelper allows some custom eval hooks. type ExecHelper interface { - GetFunc(name string) (reflect.Value, bool) - GetMapValue(receiver, key reflect.Value) (reflect.Value, bool) + GetFunc(tmpl Preparer, name string) (reflect.Value, bool) + GetMethod(tmpl Preparer, receiver reflect.Value, name string) (method reflect.Value, firstArg reflect.Value) + GetMapValue(tmpl Preparer, receiver, key reflect.Value) (reflect.Value, bool) } // Executer executes a given template. @@ -64,6 +65,7 @@ func (t *executer) Execute(p Preparer, wr io.Writer, data interface{}) error { state := &state{ helper: t.helper, + prep: p, tmpl: tmpl, wr: wr, vars: []variable{{"$", value}}, @@ -75,7 +77,6 @@ func (t *executer) Execute(p Preparer, wr io.Writer, data interface{}) error { // Prepare returns a template ready for execution. func (t *Template) Prepare() (*Template, error) { - return t, nil } @@ -95,6 +96,7 @@ func (t *Template) executeWithState(state *state, value reflect.Value) (err erro // can execute in parallel. type state struct { tmpl *Template + prep Preparer // Added for Hugo. helper ExecHelper // Added for Hugo. wr io.Writer node parse.Node // current node, for errors @@ -110,7 +112,7 @@ func (s *state) evalFunction(dot reflect.Value, node *parse.IdentifierNode, cmd var ok bool if s.helper != nil { // Added for Hugo. - function, ok = s.helper.GetFunc(name) + function, ok = s.helper.GetFunc(s.prep, name) } if !ok { @@ -148,7 +150,17 @@ func (s *state) evalField(dot reflect.Value, fieldName string, node parse.Node, if ptr.Kind() != reflect.Interface && ptr.Kind() != reflect.Ptr && ptr.CanAddr() { ptr = ptr.Addr() } + // Added for Hugo. + var first reflect.Value if method := ptr.MethodByName(fieldName); method.IsValid() { + if s.helper != nil { + method, first = s.helper.GetMethod(s.prep, ptr, fieldName) + } + + if first != zero { + return s.evalCall(dot, method, node, fieldName, args, final, first) + } + return s.evalCall(dot, method, node, fieldName, args, final) } hasArgs := len(args) > 1 || final != missingVal @@ -177,7 +189,7 @@ func (s *state) evalField(dot reflect.Value, fieldName string, node parse.Node, var result reflect.Value if s.helper != nil { // Added for Hugo. - result, _ = s.helper.GetMapValue(receiver, nameVal) + result, _ = s.helper.GetMapValue(s.prep, receiver, nameVal) } else { result = receiver.MapIndex(nameVal) } @@ -209,3 +221,79 @@ func (s *state) evalField(dot reflect.Value, fieldName string, node parse.Node, s.errorf("can't evaluate field %s in type %s", fieldName, typ) panic("not reached") } + +// evalCall executes a function or method call. If it's a method, fun already has the receiver bound, so +// it looks just like a function call. The arg list, if non-nil, includes (in the manner of the shell), arg[0] +// as the function itself. +func (s *state) evalCall(dot, fun reflect.Value, node parse.Node, name string, args []parse.Node, final reflect.Value, first ...reflect.Value) reflect.Value { + if args != nil { + args = args[1:] // Zeroth arg is function name/node; not passed to function. + } + typ := fun.Type() + numFirst := len(first) + numIn := len(args) + numFirst // // Added for Hugo + if final != missingVal { + numIn++ + } + numFixed := len(args) + len(first) + if typ.IsVariadic() { + numFixed = typ.NumIn() - 1 // last arg is the variadic one. + if numIn < numFixed { + s.errorf("wrong number of args for %s: want at least %d got %d", name, typ.NumIn()-1, len(args)) + } + } else if numIn != typ.NumIn() { + s.errorf("wrong number of args for %s: want %d got %d", name, typ.NumIn(), numIn) + } + if !goodFunc(typ) { + // TODO: This could still be a confusing error; maybe goodFunc should provide info. + s.errorf("can't call method/function %q with %d results", name, typ.NumOut()) + } + // Build the arg list. + argv := make([]reflect.Value, numIn) + // Args must be evaluated. Fixed args first. + i := len(first) + for ; i < numFixed && i < len(args)+numFirst; i++ { + argv[i] = s.evalArg(dot, typ.In(i), args[i-numFirst]) + } + // Now the ... args. + if typ.IsVariadic() { + argType := typ.In(typ.NumIn() - 1).Elem() // Argument is a slice. + for ; i < len(args)+numFirst; i++ { + argv[i] = s.evalArg(dot, argType, args[i-numFirst]) + } + + } + // Add final value if necessary. + if final != missingVal { + t := typ.In(typ.NumIn() - 1) + if typ.IsVariadic() { + if numIn-1 < numFixed { + // The added final argument corresponds to a fixed parameter of the function. + // Validate against the type of the actual parameter. + t = typ.In(numIn - 1) + } else { + // The added final argument corresponds to the variadic part. + // Validate against the type of the elements of the variadic slice. + t = t.Elem() + } + } + argv[i] = s.validateType(final, t) + } + + // Added for Hugo + for i := 0; i < len(first); i++ { + argv[i] = s.validateType(first[i], typ.In(i)) + } + + v, err := safeCall(fun, argv) + // If we have an error that is not nil, stop execution and return that + // error to the caller. + if err != nil { + s.at(node) + s.errorf("error calling %s: %v", name, err) + } + if v.Type() == reflectValueType { + v = v.Interface().(reflect.Value) + } + return v +} diff --git a/tpl/internal/go_templates/texttemplate/hugo_template_test.go b/tpl/internal/go_templates/texttemplate/hugo_template_test.go index 2424a0a484c..2601bb80e92 100644 --- a/tpl/internal/go_templates/texttemplate/hugo_template_test.go +++ b/tpl/internal/go_templates/texttemplate/hugo_template_test.go @@ -27,10 +27,18 @@ type TestStruct struct { M map[string]string } +func (t TestStruct) Hello1(arg string) string { + return arg +} + +func (t TestStruct) Hello2(arg1, arg2 string) string { + return arg1 + " " + arg2 +} + type execHelper struct { } -func (e *execHelper) GetFunc(name string) (reflect.Value, bool) { +func (e *execHelper) GetFunc(tmpl Preparer, name string) (reflect.Value, bool) { if name == "print" { return zero, false } @@ -39,11 +47,16 @@ func (e *execHelper) GetFunc(name string) (reflect.Value, bool) { }), true } -func (e *execHelper) GetMapValue(m, key reflect.Value) (reflect.Value, bool) { +func (e *execHelper) GetMapValue(tmpl Preparer, m, key reflect.Value) (reflect.Value, bool) { key = reflect.ValueOf(strings.ToLower(key.String())) return m.MapIndex(key), true } +func (e *execHelper) GetMethod(tmpl Preparer, receiver reflect.Value, name string) (method reflect.Value, firstArg reflect.Value) { + m := receiver.MethodByName("Hello2") + return m, reflect.ValueOf("v2") +} + func TestTemplateExecutor(t *testing.T) { c := qt.New(t) @@ -51,6 +64,7 @@ func TestTemplateExecutor(t *testing.T) { {{ print "foo" }} {{ printf "hugo" }} Map: {{ .M.A }} +Method: {{ .Hello1 "v1" }} `) @@ -67,5 +81,6 @@ Map: {{ .M.A }} c.Assert(got, qt.Contains, "foo") c.Assert(got, qt.Contains, "hello hugo") c.Assert(got, qt.Contains, "Map: av") + c.Assert(got, qt.Contains, "Method: v2 v1") } diff --git a/tpl/template_info.go b/tpl/template_info.go index be056695895..d38e1e822a0 100644 --- a/tpl/template_info.go +++ b/tpl/template_info.go @@ -13,6 +13,10 @@ package tpl +import ( + "github.com/gohugoio/hugo/identity" +) + // Increments on breaking changes. const TemplateVersion = 2 @@ -27,6 +31,9 @@ type Info struct { // Config extracted from template. Config Config + + // Identifies this template and its depenencies. + identity.Manager } func (info Info) IsZero() bool { diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index dd8de9067fc..b4858d52cb4 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -20,6 +20,10 @@ import ( "regexp" "time" + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/common/herrors" "strings" @@ -27,7 +31,6 @@ import ( template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate" texttemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate" - "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/tpl/tplimpl/embedded" @@ -81,6 +84,7 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { common := &templatesCommon{ nameBaseTemplateName: make(map[string]string), transformNotFound: make(map[string]bool), + identityNotFound: make(map[string][]tpl.Info), } htmlT := &htmlTemplates{ @@ -107,6 +111,8 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { }, } + textT.textTemplate.templates = textT + textT.standalone.templates = textT common.handler = h return h @@ -160,13 +166,18 @@ func (t *htmlTemplates) addTemplateIn(tt *template.Template, name, tpl string) ( typ := resolveTemplateType(name) - c, err := applyTemplateTransformersToHMLTTemplate(typ, templ) + c, err := t.handler.applyTemplateTransformersToHMLTTemplate(typ, templ) if err != nil { return nil, err } - for k := range c.notFound { + for k := range c.templateNotFound { t.transformNotFound[k] = true + t.identityNotFound[k] = append(t.identityNotFound[k], c.Info) + } + + for k := range c.identityNotFound { + t.identityNotFound[k] = append(t.identityNotFound[k], c.Info) } if typ == templateShortcode { @@ -208,7 +219,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin // * https://github.com/golang/go/issues/16101 // * https://github.com/gohugoio/hugo/issues/2549 overlayTpl = overlayTpl.Lookup(overlayTpl.Name()) - if _, err := applyTemplateTransformersToHMLTTemplate(templateUndefined, overlayTpl); err != nil { + if _, err := t.handler.applyTemplateTransformersToHMLTTemplate(templateUndefined, overlayTpl); err != nil { return err } @@ -703,36 +714,60 @@ func (t *templateHandler) loadTemplates(prefix string) error { } -func (t *templateHandler) postTransform() error { - if len(t.html.transformNotFound) == 0 && len(t.text.transformNotFound) == 0 { - return nil +func (t *templateHandler) getOrCreateTemplateInfo(name string) tpl.Info { + info, found := t.templateInfo[name] + if found { + return info + } + info = newTemplateInfo(name) + t.templateInfo[name] = info + return info +} + +func newTemplateInfo(name string) tpl.Info { + return tpl.Info{ + Manager: identity.NewIdentityManager(identity.NewPathIdentity(files.ComponentFolderLayouts, name)), + Config: tpl.DefaultConfig, } +} +func (t *templateHandler) postTransform() error { defer func() { t.text.transformNotFound = make(map[string]bool) t.html.transformNotFound = make(map[string]bool) + t.text.identityNotFound = make(map[string][]tpl.Info) + t.html.identityNotFound = make(map[string][]tpl.Info) }() for _, s := range []struct { - lookup func(name string) *parse.Tree + lookup func(name string) *templateInfoTree transformNotFound map[string]bool + identityNotFound map[string][]tpl.Info }{ // html templates - {func(name string) *parse.Tree { + {func(name string) *templateInfoTree { templ := t.html.lookup(name) if templ == nil { return nil } - return templ.Tree - }, t.html.transformNotFound}, + info := t.getOrCreateTemplateInfo(name) + return &templateInfoTree{ + info: info, + tree: templ.Tree, + } + }, t.html.transformNotFound, t.html.identityNotFound}, // text templates - {func(name string) *parse.Tree { + {func(name string) *templateInfoTree { templT := t.text.lookup(name) if templT == nil { return nil } - return templT.Tree - }, t.text.transformNotFound}, + info := t.getOrCreateTemplateInfo(name) + return &templateInfoTree{ + info: info, + tree: templT.Tree, + } + }, t.text.transformNotFound, t.text.identityNotFound}, } { for name := range s.transformNotFound { templ := s.lookup(name) @@ -743,6 +778,15 @@ func (t *templateHandler) postTransform() error { } } } + + for k, v := range s.identityNotFound { + templ := s.lookup(k) + if templ != nil && templ.info.Manager != nil { + for _, im := range v { + im.Add(templ.info) + } + } + } } return nil @@ -795,9 +839,12 @@ type templatesCommon struct { // Used to get proper filenames in errors nameBaseTemplateName map[string]string - // Holds names of the templates not found during the first AST transformation + // Holds names of the template definitions not found during the first AST transformation // pass. transformNotFound map[string]bool + + // Holds identities of templates not found during first pass. + identityNotFound map[string][]tpl.Info } func (t templatesCommon) withNewHandler(h *templateHandler) *templatesCommon { @@ -806,8 +853,9 @@ func (t templatesCommon) withNewHandler(h *templateHandler) *templatesCommon { } type textTemplate struct { - mu sync.RWMutex - t *texttemplate.Template + mu sync.RWMutex + t *texttemplate.Template + templates *textTemplates } func (t *textTemplate) Lookup(name string) (tpl.Template, bool) { @@ -831,7 +879,7 @@ func (t *textTemplate) parseIn(tt *texttemplate.Template, name, tpl string) (*te return nil, err } - if _, err := applyTemplateTransformersToTextTemplate(templateUndefined, templ); err != nil { + if _, err := t.templates.handler.applyTemplateTransformersToTextTemplate(templateUndefined, templ); err != nil { return nil, err } return templ, nil @@ -877,12 +925,12 @@ func (t *textTemplates) addTemplateIn(tt *texttemplate.Template, name, tpl strin typ := resolveTemplateType(name) - c, err := applyTemplateTransformersToTextTemplate(typ, templ) + c, err := t.handler.applyTemplateTransformersToTextTemplate(typ, templ) if err != nil { return nil, err } - for k := range c.notFound { + for k := range c.templateNotFound { t.transformNotFound[k] = true } @@ -924,7 +972,7 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin } overlayTpl = overlayTpl.Lookup(overlayTpl.Name()) - if _, err := applyTemplateTransformersToTextTemplate(templateUndefined, overlayTpl); err != nil { + if _, err := t.handler.applyTemplateTransformersToTextTemplate(templateUndefined, overlayTpl); err != nil { return err } t.overlays[name] = overlayTpl diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 31d24b71d4a..292c68aa613 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -14,8 +14,10 @@ package tplimpl import ( - template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate" + "regexp" + "strings" + template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate" texttemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" @@ -34,9 +36,10 @@ const ( ) type templateContext struct { - visited map[string]bool - notFound map[string]bool - lookupFn func(name string) *parse.Tree + visited map[string]bool + templateNotFound map[string]bool + identityNotFound map[string]bool + lookupFn func(name string) *templateInfoTree // The last error encountered. err error @@ -53,7 +56,7 @@ type templateContext struct { returnNode *parse.CommandNode } -func (c templateContext) getIfNotVisited(name string) *parse.Tree { +func (c templateContext) getIfNotVisited(name string) *templateInfoTree { if c.visited[name] { return nil } @@ -63,59 +66,96 @@ func (c templateContext) getIfNotVisited(name string) *parse.Tree { // This may be a inline template defined outside of this file // and not yet parsed. Unusual, but it happens. // Store the name to try again later. - c.notFound[name] = true + c.templateNotFound[name] = true } return templ } -func newTemplateContext(lookupFn func(name string) *parse.Tree) *templateContext { +func newTemplateContext(info tpl.Info, lookupFn func(name string) *templateInfoTree) *templateContext { + if info.Manager == nil { + panic("identity manager not set") + } return &templateContext{ - Info: tpl.Info{Config: tpl.DefaultConfig}, - lookupFn: lookupFn, - visited: make(map[string]bool), - notFound: make(map[string]bool)} + Info: info, + lookupFn: lookupFn, + visited: make(map[string]bool), + templateNotFound: make(map[string]bool), + identityNotFound: make(map[string]bool), + } +} + +func createParseTreeLookup(templ *template.Template) func(nn string) *templateInfoTree { + return createParseTreeLookupFor(templ, func(name string) tpl.Info { return newTemplateInfo(name) }) + } -func createParseTreeLookup(templ *template.Template) func(nn string) *parse.Tree { - return func(nn string) *parse.Tree { +func createParseTreeLookupFor(templ *template.Template, infoFn func(name string) tpl.Info) func(nn string) *templateInfoTree { + return func(nn string) *templateInfoTree { tt := templ.Lookup(nn) if tt != nil { - return tt.Tree + return &templateInfoTree{ + tree: tt.Tree, + info: infoFn(nn), + } } return nil } } +func (t *templateHandler) createParseTreeLookup(templ *template.Template) func(nn string) *templateInfoTree { + return createParseTreeLookupFor(templ, func(name string) tpl.Info { return t.templateInfo[name] }) +} -func applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (*templateContext, error) { - return applyTemplateTransformers(typ, templ.Tree, createParseTreeLookup(templ)) +func (t *templateHandler) applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (*templateContext, error) { + ti := &templateInfoTree{ + tree: templ.Tree, + info: t.getOrCreateTemplateInfo(templ.Name()), + } + return applyTemplateTransformers(typ, ti, t.createParseTreeLookup(templ)) } -func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (*templateContext, error) { - return applyTemplateTransformers(typ, templ.Tree, - func(nn string) *parse.Tree { +func (t *templateHandler) applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (*templateContext, error) { + ti := &templateInfoTree{ + tree: templ.Tree, + info: t.getOrCreateTemplateInfo(templ.Name()), + } + + return applyTemplateTransformers(typ, ti, + func(nn string) *templateInfoTree { tt := templ.Lookup(nn) if tt != nil { - return tt.Tree + return &templateInfoTree{ + tree: tt.Tree, + info: t.getOrCreateTemplateInfo(nn), + } } return nil }) } -func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn func(name string) *parse.Tree) (*templateContext, error) { +type templateInfoTree struct { + info tpl.Info + tree *parse.Tree +} + +func applyTemplateTransformers( + typ templateType, + templ *templateInfoTree, + lookupFn func(name string) *templateInfoTree) (*templateContext, error) { + if templ == nil { return nil, errors.New("expected template, but none provided") } - c := newTemplateContext(lookupFn) + c := newTemplateContext(templ.info, lookupFn) c.typ = typ - _, err := c.applyTransformations(templ.Root) + _, err := c.applyTransformations(templ.tree.Root) if err == nil && c.returnNode != nil { // This is a partial with a return statement. c.Info.HasReturn = true - templ.Root = c.wrapInPartialReturnWrapper(templ.Root) + templ.tree.Root = c.wrapInPartialReturnWrapper(templ.tree.Root) } return c, err @@ -125,7 +165,10 @@ const ( partialReturnWrapperTempl = `{{ $_hugo_dot := $ }}{{ $ := .Arg }}{{ with .Arg }}{{ $_hugo_dot.Set ("PLACEHOLDER") }}{{ end }}` ) -var partialReturnWrapper *parse.ListNode +var ( + partialReturnWrapper *parse.ListNode + hugoInfoTemplate *parse.ActionNode +) func init() { templ, err := texttemplate.New("").Parse(partialReturnWrapperTempl) @@ -133,6 +176,7 @@ func init() { panic(err) } partialReturnWrapper = templ.Tree.Root + } func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.ListNode { @@ -156,6 +200,7 @@ func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.L // getif works slightly different than the Go built-in in that it also // considers any IsZero methods on the values (as in time.Time). // See https://github.com/gohugoio/hugo/issues/5738 +// TODO(bep) get rid of this. func (c *templateContext) wrapWithGetIf(p *parse.PipeNode) { if len(p.Cmds) == 0 { return @@ -176,9 +221,9 @@ func (c *templateContext) wrapWithGetIf(p *parse.PipeNode) { } // applyTransformations do 3 things: -// 1) Make all .Params.CamelCase and similar into lowercase. -// 2) Wraps every with and if pipe in getif -// 3) Collects some information about the template content. +// 1) Wraps every with and if pipe in getif +// 2) Parses partial return statement. +// 3) Tracks template (partial) dependencies and some other info. func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { switch x := n.(type) { case *parse.ListNode: @@ -198,7 +243,7 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { case *parse.TemplateNode: subTempl := c.getIfNotVisited(x.Name) if subTempl != nil { - c.applyTransformationsToNodes(subTempl.Root) + c.applyTransformationsToNodes(subTempl.tree.Root) } case *parse.PipeNode: c.collectConfig(x) @@ -210,6 +255,7 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } case *parse.CommandNode: + c.collectPartialInfo(x) c.collectInner(x) keep := c.collectReturnNode(x) @@ -311,6 +357,38 @@ func (c *templateContext) collectInner(n *parse.CommandNode) { } +var partialRe = regexp.MustCompile(`^partial(Cached)?$|^partials\.Include(Cached)?$`) + +func (c *templateContext) collectPartialInfo(x *parse.CommandNode) { + if len(x.Args) < 2 { + return + } + + first := x.Args[0] + var id string + switch v := first.(type) { + case *parse.IdentifierNode: + id = v.Ident + case *parse.ChainNode: + id = v.String() + } + + if partialRe.MatchString(id) { + partialName := strings.Trim(x.Args[1].String(), "\"") + if !strings.Contains(partialName, ".") { + partialName += ".html" + } + partialName = "partials/" + partialName + info := c.lookupFn(partialName) + if info != nil { + c.Info.Add(info.info) + } else { + // Delay for later + c.identityNotFound[partialName] = true + } + } +} + func (c *templateContext) collectReturnNode(n *parse.CommandNode) bool { if c.typ != templatePartial || c.returnNode != nil { return true diff --git a/tpl/tplimpl/template_ast_transformers_test.go b/tpl/tplimpl/template_ast_transformers_test.go index 0dc91ac3269..d69c9e2fd2b 100644 --- a/tpl/tplimpl/template_ast_transformers_test.go +++ b/tpl/tplimpl/template_ast_transformers_test.go @@ -20,9 +20,16 @@ import ( "testing" "time" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/tpl" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) - qt "github.com/frankban/quicktest" +var eq = qt.CmpEquals( + cmp.Comparer(func(i1, i2 tpl.Info) bool { + return cmp.Equal(i1, i2, cmpopts.IgnoreFields(tpl.Info{}, "Manager")) + }), ) // Issue #2927 @@ -33,7 +40,7 @@ func TestTransformRecursiveTemplate(t *testing.T) { {{ define "menu-nodes" }} {{ template "menu-node" }} {{ end }} -{{ define "menu-node" }} +{{ define "menu-nßode" }} {{ template "menu-node" }} {{ end }} {{ template "menu-nodes" }} @@ -42,7 +49,10 @@ func TestTransformRecursiveTemplate(t *testing.T) { templ, err := template.New("foo").Parse(recursive) c.Assert(err, qt.IsNil) - ctx := newTemplateContext(createParseTreeLookup(templ)) + ctx := newTemplateContext( + newTemplateInfo("test"), + createParseTreeLookup(templ), + ) ctx.applyTransformations(templ.Tree.Root) } @@ -80,13 +90,10 @@ func TestInsertIsZeroFunc(t *testing.T) { {{ with .TimeZero }}.TimeZero1 with: {{ . }}{{ else }}.TimeZero1 with: FALSE{{ end }} {{ template "mytemplate" . }} {{ if .T.NonEmptyInterfaceTypedNil }}.NonEmptyInterfaceTypedNil: TRUE{{ else }}.NonEmptyInterfaceTypedNil: FALSE{{ end }} - {{ template "other-file-template" . }} - {{ define "mytemplate" }} {{ if .TimeZero }}.TimeZero1: mytemplate: TRUE{{ else }}.TimeZero1: mytemplate: FALSE{{ end }} {{ end }} - ` // https://github.com/gohugoio/hugo/issues/5865 @@ -110,12 +117,10 @@ func TestInsertIsZeroFunc(t *testing.T) { c.Assert(h.MarkReady(), qt.IsNil) for _, name := range []string{"mytemplate.html", "mytexttemplate.txt"} { + var sb strings.Builder tt, _ := d.Tmpl.Lookup(name) - sb := &strings.Builder{} - - err := d.Tmpl.Execute(tt, sb, ctx) + err := h.Execute(tt, &sb, ctx) c.Assert(err, qt.IsNil) - result := sb.String() c.Assert(result, qt.Contains, ".True: TRUE") @@ -163,11 +168,12 @@ func TestCollectInfo(t *testing.T) { templ, err := template.New("foo").Funcs(funcs).Parse(test.tplString) c.Assert(err, qt.IsNil) - ctx := newTemplateContext(createParseTreeLookup(templ)) + ctx := newTemplateContext( + newTemplateInfo("test"), createParseTreeLookup(templ)) ctx.typ = templateShortcode ctx.applyTransformations(templ.Tree.Root) - c.Assert(ctx.Info, qt.Equals, test.expected) + c.Assert(ctx.Info, eq, test.expected) }) } @@ -205,7 +211,7 @@ func TestPartialReturn(t *testing.T) { templ, err := template.New("foo").Funcs(funcs).Parse(test.tplString) c.Assert(err, qt.IsNil) - _, err = applyTemplateTransformers(templatePartial, templ.Tree, createParseTreeLookup(templ)) + _, err = applyTemplateTransformers(templatePartial, &templateInfoTree{tree: templ.Tree, info: newTemplateInfo("test")}, createParseTreeLookup(templ)) // Just check that it doesn't fail in this test. We have functional tests // in hugoblib. diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index 2098732f6bc..e152725ac1a 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -19,6 +19,8 @@ import ( "reflect" "strings" + "github.com/gohugoio/hugo/tpl" + "github.com/gohugoio/hugo/common/maps" template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate" @@ -62,14 +64,14 @@ type templateExecHelper struct { funcs map[string]reflect.Value } -func (t *templateExecHelper) GetFunc(name string) (reflect.Value, bool) { +func (t *templateExecHelper) GetFunc(tmpl texttemplate.Preparer, name string) (reflect.Value, bool) { if fn, found := t.funcs[name]; found { return fn, true } return zero, false } -func (t *templateExecHelper) GetMapValue(receiver, key reflect.Value) (reflect.Value, bool) { +func (t *templateExecHelper) GetMapValue(tmpl texttemplate.Preparer, receiver, key reflect.Value) (reflect.Value, bool) { if params, ok := receiver.Interface().(maps.Params); ok { // Case insensitive. keystr := strings.ToLower(key.String()) @@ -85,6 +87,16 @@ func (t *templateExecHelper) GetMapValue(receiver, key reflect.Value) (reflect.V return v, v.IsValid() } +func (t *templateExecHelper) GetMethod(tmpl texttemplate.Preparer, receiver reflect.Value, name string) (method reflect.Value, firstArg reflect.Value) { + if info, ok := tmpl.(tpl.TemplateInfoProvider); ok { + if m := receiver.MethodByName(name + "WithTemplateInfo"); m.IsValid() { + return m, reflect.ValueOf(info) + } + } + + return receiver.MethodByName(name), zero +} + func newTemplateExecuter(d *deps.Deps) (texttemplate.Executer, map[string]reflect.Value) { funcs := createFuncMap(d) funcsv := make(map[string]reflect.Value)