From 496b826caac93cc4b6c2c5a8fbae669e6d66f59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 20 Jan 2024 11:14:41 +0100 Subject: [PATCH] Add a option to duplicate non-page (e.g. images) resources across languages Which will be default behaviour for all other than Goldmark, for which there is a config option: ``` [markup.goldmark] duplicateResourceFiles = true ``` Fixes #11904 --- common/paths/pathparser.go | 14 ++ common/paths/pathparser_test.go | 2 + config/commonConfig.go | 8 +- hugolib/content_map.go | 29 ++- hugolib/content_map_page.go | 217 ++++++++++++++++------ hugolib/content_map_test.go | 6 +- hugolib/doctree/dimensions.go | 30 ++- hugolib/doctree/dimensions_test.go | 15 +- hugolib/doctree/nodeshiftree_test.go | 130 ++++++------- hugolib/doctree/nodeshifttree.go | 78 ++++---- hugolib/doctree/support.go | 3 +- hugolib/hugo_sites.go | 2 +- hugolib/page.go | 3 +- hugolib/page__tree.go | 2 +- hugolib/page_test.go | 2 +- hugolib/pagebundler_test.go | 73 +++++++- hugolib/site.go | 2 +- hugolib/site_new.go | 24 +-- hugolib/site_render.go | 4 +- markup/goldmark/goldmark_config/config.go | 7 +- markup/markup.go | 6 + resources/resource.go | 11 ++ resources/resource/resources.go | 17 ++ resources/resource/resourcetypes.go | 10 + resources/resource_metadata.go | 7 +- resources/transform.go | 6 + 26 files changed, 480 insertions(+), 228 deletions(-) diff --git a/common/paths/pathparser.go b/common/paths/pathparser.go index e376a6677d1..842d9307b06 100644 --- a/common/paths/pathparser.go +++ b/common/paths/pathparser.go @@ -361,6 +361,15 @@ func (p *Path) PathNoIdentifier() string { return p.base(false, false) } +// PathRel returns the path relativeto the given owner. +func (p *Path) PathRel(owner *Path) string { + ob := owner.Base() + if !strings.HasSuffix(ob, "/") { + ob += "/" + } + return strings.TrimPrefix(p.Path(), ob) +} + // BaseRel returns the base path relative to the given owner. func (p *Path) BaseRel(owner *Path) string { ob := owner.Base() @@ -378,6 +387,11 @@ func (p *Path) Base() string { return p.base(!p.isContentPage(), p.IsBundle()) } +// BaseNoLeadingSlash returns the base path without the leading slash. +func (p *Path) BaseNoLeadingSlash() string { + return p.Base()[1:] +} + func (p *Path) base(preserveExt, isBundle bool) string { if len(p.identifiers) == 0 { return p.norm(p.s) diff --git a/common/paths/pathparser_test.go b/common/paths/pathparser_test.go index 8c01e1a087c..3546b66050e 100644 --- a/common/paths/pathparser_test.go +++ b/common/paths/pathparser_test.go @@ -121,6 +121,7 @@ func TestParse(t *testing.T) { func(c *qt.C, p *Path) { c.Assert(p.Name(), qt.Equals, "b.md") c.Assert(p.Base(), qt.Equals, "/a/b") + c.Assert(p.BaseNoLeadingSlash(), qt.Equals, "a/b") c.Assert(p.Section(), qt.Equals, "a") c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b") @@ -160,6 +161,7 @@ func TestParse(t *testing.T) { c.Assert(p.NameNoLang(), qt.Equals, "b.a.b.txt") c.Assert(p.Identifiers(), qt.DeepEquals, []string{"txt", "no"}) c.Assert(p.Base(), qt.Equals, "/a/b.a.b.txt") + c.Assert(p.BaseNoLeadingSlash(), qt.Equals, "a/b.a.b.txt") c.Assert(p.PathNoLang(), qt.Equals, "/a/b.a.b.txt") c.Assert(p.Ext(), qt.Equals, "txt") c.Assert(p.PathNoIdentifier(), qt.Equals, "/a/b.a.b") diff --git a/config/commonConfig.go b/config/commonConfig.go index b10f7dcead4..6ca061093c5 100644 --- a/config/commonConfig.go +++ b/config/commonConfig.go @@ -94,7 +94,13 @@ var defaultBuild = BuildConfig{ // BuildConfig holds some build related configuration. type BuildConfig struct { - UseResourceCacheWhen string // never, fallback, always. Default is fallback + // When to use the resource file cache. + // One of never, fallback, always. Default is fallback + UseResourceCacheWhen string + + // When enabled, will duplicate bundled resource files across languages that + // doesn't have a translated version. + DuplicateResourceFiles bool // When enabled, will collect and write a hugo_stats.json with some build // related aggregated data (e.g. CSS class names). diff --git a/hugolib/content_map.go b/hugolib/content_map.go index 8322b6958ce..95c1e860079 100644 --- a/hugolib/content_map.go +++ b/hugolib/content_map.go @@ -58,6 +58,19 @@ type resourceSource struct { r resource.Resource } +func (r resourceSource) clone() *resourceSource { + r.r = nil + return &r +} + +func (r *resourceSource) LangIndex() int { + if r.r != nil && r.isPage() { + return r.r.(*pageState).s.languagei + } + + return r.fi.Meta().LangIndex +} + func (r *resourceSource) MarkStale() { resource.MarkStale(r.r) } @@ -151,16 +164,6 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo) error { commit := tree.Lock(true) defer commit() - var resources resourceSources - n, ok := tree.GetRaw(key) - - if ok { - resources = n.(resourceSources) - } else { - resources = make(resourceSources, len(m.s.h.Sites)) - tree.Insert(key, resources) - } - r := func() (hugio.ReadSeekCloser, error) { return fim.Meta().Open() } @@ -184,11 +187,7 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo) error { rs = &resourceSource{path: pi, opener: r, fi: fim} } - i := fim.Meta().LangIndex - if r := resources[i]; r != nil && r.r != nil { - resource.MarkStale(r.r) - } - resources[i] = rs + tree.InsertIntoValuesDimension(key, rs) return nil } diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index 469c0c1a8ff..2f8b9f88fb5 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -101,10 +101,6 @@ type pageMap struct { cfg contentMapConfig } -const ( - pageTreeDimensionLanguage = iota -) - // pageTrees holds pages and resources in a tree structure for all sites/languages. // Eeach site gets its own tree set via the Shape method. type pageTrees struct { @@ -210,7 +206,7 @@ func (m *pageMap) forEachPage(include predicate.P[*pageState], fn func(p *pageSt w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: m.treePages, LockType: doctree.LockTypeRead, - Handle: func(key string, n contentNodeI) (bool, error) { + Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if p, ok := n.(*pageState); ok && include(p) { if terminate, err := fn(p); terminate || err != nil { return terminate, err @@ -237,7 +233,7 @@ func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.P[*pageSta w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: m.treeResources, LockType: doctree.LockTypeRead, - Handle: func(key string, n contentNodeI) (bool, error) { + Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if rs, ok := n.(*resourceSource); ok { if p, ok := rs.r.(*pageState); ok && include(p) { if terminate, err := fn(p); terminate || err != nil { @@ -277,7 +273,7 @@ func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages { w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: m.treePages, Prefix: prefix, - Handle: func(key string, n contentNodeI) (bool, error) { + Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if q.Recursive { if p, ok := n.(*pageState); ok && include(p) { pas = append(pas, p) @@ -398,7 +394,7 @@ func (m *pageMap) forEachResourceInPage( ps *pageState, lockType doctree.LockType, exact bool, - handle func(resourceKey string, n contentNodeI) (bool, error), + handle func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error), ) error { keyPage := ps.Path() if keyPage == "/" { @@ -414,7 +410,7 @@ func (m *pageMap) forEachResourceInPage( Exact: exact, } - rw.Handle = func(resourceKey string, n contentNodeI) (bool, error) { + rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if isBranch { ownerKey, _ := m.treePages.LongestPrefixAll(resourceKey) if ownerKey != keyPage { @@ -423,7 +419,7 @@ func (m *pageMap) forEachResourceInPage( return false, nil } } - return handle(resourceKey, n) + return handle(resourceKey, n, match) } return rw.Walk(context.Background()) @@ -431,7 +427,7 @@ func (m *pageMap) forEachResourceInPage( func (m *pageMap) getResourcesForPage(ps *pageState) (resource.Resources, error) { var res resource.Resources - m.forEachResourceInPage(ps, doctree.LockTypeNone, false, func(resourceKey string, n contentNodeI) (bool, error) { + m.forEachResourceInPage(ps, doctree.LockTypeNone, false, func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { rs := n.(*resourceSource) if rs.r != nil { res = append(res, rs.r) @@ -575,9 +571,11 @@ func (n contentNodeIs) MarkStale() { } } -type contentNodeShifter struct{} +type contentNodeShifter struct { + numLanguages int +} -func (s *contentNodeShifter) Delete(n contentNodeI, dimension []int) (bool, bool) { +func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (bool, bool) { lidx := dimension[0] switch v := n.(type) { case contentNodeIs: @@ -604,6 +602,9 @@ func (s *contentNodeShifter) Delete(n contentNodeI, dimension []int) (bool, bool } } return wasDeleted, isEmpty + case *resourceSource: + resource.MarkStale(v) + return true, true case *pageState: resource.MarkStale(v) return true, true @@ -612,40 +613,52 @@ func (s *contentNodeShifter) Delete(n contentNodeI, dimension []int) (bool, bool } } -func (s *contentNodeShifter) Shift(n contentNodeI, dimension []int, exact bool) (contentNodeI, bool) { +func (s *contentNodeShifter) Shift(n contentNodeI, dimension doctree.Dimension, exact bool) (contentNodeI, bool, doctree.DimensionFlag) { lidx := dimension[0] + // How accurate is the match. + accuracy := doctree.DimensionLanguage switch v := n.(type) { case contentNodeIs: if len(v) == 0 { panic("empty contentNodeIs") } vv := v[lidx] - return vv, vv != nil + if vv != nil { + return vv, true, accuracy + } + return nil, false, 0 case resourceSources: vv := v[lidx] if vv != nil { - return vv, true + return vv, true, doctree.DimensionLanguage } if exact { - return nil, false + return nil, false, 0 } // For non content resources, pick the first match. for _, vv := range v { if vv != nil { if vv.isPage() { - return nil, false + return nil, false, 0 } - return vv, true + return vv, true, 0 } } + case *resourceSource: + if v.LangIndex() == lidx { + return v, true, doctree.DimensionLanguage + } + if !v.isPage() && !exact { + return v, true, 0 + } case *pageState: if v.s.languagei == lidx { - return n, true + return n, true, doctree.DimensionLanguage } default: panic(fmt.Sprintf("unknown type %T", n)) } - return nil, false + return nil, false, 0 } func (s *contentNodeShifter) All(n contentNodeI) []contentNodeI { @@ -665,23 +678,85 @@ func (s *contentNodeShifter) Dimension(n contentNodeI, d int) []contentNodeI { return s.All(n) } -func (s *contentNodeShifter) Insert(old, new contentNodeI) (contentNodeI, bool) { - newp, ok := new.(*pageState) - if !ok { - panic(fmt.Sprintf("unknown type %T", new)) +func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) contentNodeI { + langi := dimension[doctree.DimensionLanguage.Index()] + switch vv := old.(type) { + case *pageState: + newp, ok := new.(*pageState) + if !ok { + panic(fmt.Sprintf("unknown type %T", new)) + } + if vv.s.languagei == newp.s.languagei && newp.s.languagei == langi { + return new + } + is := make(contentNodeIs, s.numLanguages) + is[vv.s.languagei] = old + is[langi] = new + return is + case contentNodeIs: + vv[langi] = new + return vv + case resourceSources: + vv[langi] = new.(*resourceSource) + return vv + case *resourceSource: + newp, ok := new.(*resourceSource) + if !ok { + panic(fmt.Sprintf("unknown type %T", new)) + } + if vv.LangIndex() == newp.LangIndex() && newp.LangIndex() == langi { + return new + } + rs := make(resourceSources, s.numLanguages) + rs[vv.LangIndex()] = vv + rs[langi] = newp + return rs + + default: + panic(fmt.Sprintf("unknown type %T", old)) } +} + +func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI { switch vv := old.(type) { case *pageState: - if vv.Lang() == newp.Lang() { - return new, true + newp, ok := new.(*pageState) + if !ok { + panic(fmt.Sprintf("unknown type %T", new)) + } + if vv.s.languagei == newp.s.languagei { + return new } - is := make(contentNodeIs, len(newp.s.h.Sites)) + is := make(contentNodeIs, s.numLanguages) is[newp.s.languagei] = new is[vv.s.languagei] = old - return is, true + return is case contentNodeIs: + newp, ok := new.(*pageState) + if !ok { + panic(fmt.Sprintf("unknown type %T", new)) + } vv[newp.s.languagei] = new - return vv, true + return vv + case *resourceSource: + newp, ok := new.(*resourceSource) + if !ok { + panic(fmt.Sprintf("unknown type %T", new)) + } + if vv.LangIndex() == newp.LangIndex() { + return new + } + rs := make(resourceSources, s.numLanguages) + rs[newp.LangIndex()] = newp + rs[vv.LangIndex()] = vv + return rs + case resourceSources: + newp, ok := new.(*resourceSource) + if !ok { + panic(fmt.Sprintf("unknown type %T", new)) + } + vv[newp.LangIndex()] = newp + return vv default: panic(fmt.Sprintf("unknown type %T", old)) } @@ -726,7 +801,7 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) * w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: m.treePages, LockType: doctree.LockTypeRead, - Handle: func(s string, n contentNodeI) (bool, error) { + Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { p := n.(*pageState) if p.File() != nil { add(p.File().FileInfo().Meta().PathInfo.BaseNameNoIdentifier(), p) @@ -791,7 +866,7 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) { resourceWalker := pageWalker.Extend() resourceWalker.Tree = m.treeResources - pageWalker.Handle = func(keyPage string, n contentNodeI) (bool, error) { + pageWalker.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { level := strings.Count(keyPage, "/") if level > maxLevel { return false, nil @@ -820,7 +895,7 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) { prevKey = keyPage resourceWalker.Prefix = keyPage + "/" - resourceWalker.Handle = func(ss string, n contentNodeI) (bool, error) { + resourceWalker.Handle = func(ss string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if isBranch { ownerKey, _ := pageWalker.Tree.LongestPrefix(ss, true, nil) if ownerKey != keyPage { @@ -1046,7 +1121,7 @@ func (sa *sitePagesAssembler) applyAggregates() error { rw.Tree = sa.pageMap.treeResources sa.lastmod = time.Time{} - pw.Handle = func(keyPage string, n contentNodeI) (bool, error) { + pw.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { pageBundle := n.(*pageState) if pageBundle.Kind() == kinds.KindTerm { @@ -1135,7 +1210,7 @@ func (sa *sitePagesAssembler) applyAggregates() error { isBranch := n.isContentNodeBranch() rw.Prefix = keyPage + "/" - rw.Handle = func(resourceKey string, n contentNodeI) (bool, error) { + rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if isBranch { ownerKey, _ := pw.Tree.LongestPrefix(resourceKey, true, nil) if ownerKey != keyPage { @@ -1197,7 +1272,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error { Prefix: key, // We also want to include the root taxonomy nodes, so no trailing slash. LockType: doctree.LockTypeRead, WalkContext: walkContext, - Handle: func(s string, n contentNodeI) (bool, error) { + Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { p := n.(*pageState) if p.Kind() != kinds.KindTerm { // The other kinds were handled in applyAggregates. @@ -1283,7 +1358,7 @@ func (sa *sitePagesAssembler) assembleTermsAndTranslations() error { w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: pages, LockType: lockType, - Handle: func(s string, n contentNodeI) (bool, error) { + Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { ps := n.(*pageState) if ps.m.noLink() { @@ -1333,7 +1408,7 @@ func (sa *sitePagesAssembler) assembleTermsAndTranslations() error { if err != nil { return false, err } - pages.Insert(pi.Base(), n) + pages.InsertIntoValuesDimension(pi.Base(), n) term = pages.Get(pi.Base()) } @@ -1359,44 +1434,65 @@ func (sa *sitePagesAssembler) assembleTermsAndTranslations() error { } func (sa *sitePagesAssembler) assembleResources() error { - pages := sa.pageMap.treePages + pagesTree := sa.pageMap.treePages + resourcesTree := sa.pageMap.treeResources lockType := doctree.LockTypeWrite w := &doctree.NodeShiftTreeWalker[contentNodeI]{ - Tree: pages, + Tree: pagesTree, LockType: lockType, - Handle: func(s string, n contentNodeI) (bool, error) { + Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { ps := n.(*pageState) // Prepare resources for this page. ps.shiftToOutputFormat(true, 0) targetPaths := ps.targetPaths() + baseTarget := targetPaths.SubResourceBaseTarget + duplicateResourceFiles := true + if ps.s.ContentSpec.Converters.IsGoldmark(ps.m.markup) { + duplicateResourceFiles = ps.s.ContentSpec.Converters.GetMarkupConfig().Goldmark.DuplicateResourceFiles + } + sa.pageMap.forEachResourceInPage( ps, lockType, - true, // No shifting, we only want resources for this dimension. - func(resourceKey string, n contentNodeI) (bool, error) { + !duplicateResourceFiles, // wheter to exclude alternative language matches. + func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { rs := n.(*resourceSource) + if !match.Has(doctree.DimensionLanguage) { + // We got an alternative language version. + // Clone this and insert it into the tree. + rs = rs.clone() + resourcesTree.InsertIntoCurrentDimension(resourceKey, rs) + } if rs.r != nil { return false, nil } + + relPathOriginal := rs.path.PathRel(ps.m.pathInfo) relPath := rs.path.BaseRel(ps.m.pathInfo) - baseTarget := targetPaths.SubResourceBaseTarget + var targetBasePaths []string if ps.s.Conf.IsMultihost() { + baseTarget = targetPaths.SubResourceBaseLink // In multihost we need to publish to the lang sub folder, // and also to all languages that does not have this bundled resource. - targetBasePaths = []string{ps.Lang()} - // TODO(bep) improve. - var sources resourceSources - if v, ok := sa.pageMap.treeResources.GetRaw(resourceKey); ok { - sources = v.(resourceSources) - for i, s := range sources { - if s == nil { - targetBasePaths = append(targetBasePaths, ps.s.h.Sites[i].Lang()) + targetBasePaths = []string{ps.s.GetTargetLanguageBasePath()} + if v, ok := resourcesTree.GetRaw(resourceKey); ok { + switch vv := v.(type) { + case resourceSources: + for i, s := range vv { + if s == nil { + targetBasePaths = append(targetBasePaths, ps.s.h.Sites[i].GetTargetLanguageBasePath()) + } + } + case *resourceSource: + for i, s := range ps.s.h.Sites { + if i != vv.LangIndex() { + targetBasePaths = append(targetBasePaths, s.GetTargetLanguageBasePath()) + } } } } - baseTarget = targetPaths.SubResourceBaseLink } @@ -1404,11 +1500,12 @@ func (sa *sitePagesAssembler) assembleResources() error { OpenReadSeekCloser: rs.opener, Path: rs.path, GroupIdentity: rs.path, - TargetPath: relPath, + TargetPath: relPathOriginal, // Use the original path for the target path, so the links can be guessed. TargetBasePaths: targetBasePaths, BasePathRelPermalink: targetPaths.SubResourceBaseLink, BasePathTargetPath: baseTarget, Name: relPath, + NameOriginal: relPathOriginal, LazyPublish: !ps.m.buildConfig.PublishResources, } r, err := ps.m.s.ResourceSpec.NewResource(rd) @@ -1467,7 +1564,7 @@ func (sa *sitePagesAssembler) removeShouldNotBuild() error { w := &doctree.NodeShiftTreeWalker[contentNodeI]{ LockType: doctree.LockTypeRead, Tree: sa.pageMap.treePages, - Handle: func(key string, n contentNodeI) (bool, error) { + Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { p := n.(*pageState) if !s.shouldBuild(p) { switch p.Kind() { @@ -1524,7 +1621,7 @@ func (sa *sitePagesAssembler) addStandalonePages() error { p, _ := s.h.newPage(m) - tree.Insert(key, p) + tree.InsertIntoValuesDimension(key, p) } addStandalone("/404", kinds.KindStatus404, output.HTTPStatusHTMLFormat) @@ -1564,7 +1661,7 @@ func (sa *sitePagesAssembler) addMissingRootSections() error { w = &doctree.NodeShiftTreeWalker[contentNodeI]{ LockType: doctree.LockTypeWrite, Tree: sa.pageMap.treePages, - Handle: func(s string, n contentNodeI) (bool, error) { + Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if n == nil { panic("n is nil") } @@ -1603,7 +1700,7 @@ func (sa *sitePagesAssembler) addMissingRootSections() error { if err != nil { return false, err } - w.Tree.Insert(pth.Base(), ps) + w.Tree.InsertIntoValuesDimension(pth.Base(), ps) } // /a/b, we don't need to walk deeper. @@ -1657,7 +1754,7 @@ func (sa *sitePagesAssembler) addMissingTaxonomies() error { singular: viewName.singular, } p, _ := sa.h.newPage(m) - tree.Insert(key, p) + tree.InsertIntoValuesDimension(key, p) } } @@ -1678,7 +1775,7 @@ func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) error { Tree: m.treePages, Prefix: paths.AddTrailingSlash(key), LockType: doctree.LockTypeRead, - Handle: func(s string, n contentNodeI) (bool, error) { + Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { p := n.(*pageState) plural := p.Section() diff --git a/hugolib/content_map_test.go b/hugolib/content_map_test.go index 33ab6a40d4e..a41b2aae916 100644 --- a/hugolib/content_map_test.go +++ b/hugolib/content_map_test.go @@ -234,9 +234,9 @@ Data en ` b := Test(t, files) - b.AssertFileContent("public/fr/index.html", "fr: /fr/bundle/data.txt|Data fr") - b.AssertFileContent("public/en/index.html", "en: /en/bundle/data.txt|Data en") - b.AssertFileContent("public/de/index.html", "de: /fr/bundle/data.txt|Data fr") + b.AssertFileContent("public/fr/index.html", "fr: /fr/bundle/data.fr.txt|Data fr") + b.AssertFileContent("public/en/index.html", "en: /en/bundle/data.en.txt|Data en") + b.AssertFileContent("public/de/index.html", "de: /fr/bundle/data.fr.txt|Data fr") } func TestBundleMultipleContentPageWithSamePath(t *testing.T) { diff --git a/hugolib/doctree/dimensions.go b/hugolib/doctree/dimensions.go index 8724c58e085..bcc3cae00ef 100644 --- a/hugolib/doctree/dimensions.go +++ b/hugolib/doctree/dimensions.go @@ -13,5 +13,31 @@ package doctree -// Dimensions is a row in the Hugo build matrix which currently has one dimension: language. -type Dimensions []int +const ( + // Language is currently the only dimension in the Hugo build matrix. + DimensionLanguage DimensionFlag = 1 << iota +) + +// Dimension is a row in the Hugo build matrix which currently has one value: language. +type Dimension [1]int + +// DimensionFlag is a flag in the Hugo build matrix. +type DimensionFlag byte + +// Has returns whether the given flag is set. +func (d DimensionFlag) Has(o DimensionFlag) bool { + return d&o == o +} + +// Set sets the given flag. +func (d DimensionFlag) Set(o DimensionFlag) DimensionFlag { + return d | o +} + +// Index returns this flag's index in the Dimensions array. +func (d DimensionFlag) Index() int { + if d == 0 { + panic("dimension flag not set") + } + return int(d - 1) +} diff --git a/hugolib/doctree/dimensions_test.go b/hugolib/doctree/dimensions_test.go index 8b13b01c778..598f22a2d63 100644 --- a/hugolib/doctree/dimensions_test.go +++ b/hugolib/doctree/dimensions_test.go @@ -19,8 +19,19 @@ import ( qt "github.com/frankban/quicktest" ) -func TestDim(t *testing.T) { +func TestDimensionFlag(t *testing.T) { c := qt.New(t) - c.Assert(true, qt.Equals, true) + var zero DimensionFlag + var d DimensionFlag + var o DimensionFlag = 1 + var p DimensionFlag = 12 + + c.Assert(d.Has(o), qt.Equals, false) + d = d.Set(o) + c.Assert(d.Has(o), qt.Equals, true) + c.Assert(d.Has(d), qt.Equals, true) + c.Assert(func() { zero.Index() }, qt.PanicMatches, "dimension flag not set") + c.Assert(DimensionLanguage.Index(), qt.Equals, 0) + c.Assert(p.Index(), qt.Equals, 11) } diff --git a/hugolib/doctree/nodeshiftree_test.go b/hugolib/doctree/nodeshiftree_test.go index cdea15fe4f3..79db1a24d8b 100644 --- a/hugolib/doctree/nodeshiftree_test.go +++ b/hugolib/doctree/nodeshiftree_test.go @@ -33,7 +33,7 @@ var eq = qt.CmpEquals( return true } - return n1.ID == n2.ID && n1.Lang == n2.Lang && n1.Role == n2.Role + return n1.ID == n2.ID && n1.Lang == n2.Lang }), ) @@ -42,31 +42,24 @@ func TestTree(t *testing.T) { zeroZero := doctree.New( doctree.Config[*testValue]{ - MatrixRow: []int{0, 0}, - Shifter: &testShifter{}, + Shifter: &testShifter{}, }, ) a := &testValue{ID: "/a"} - zeroZero.Insert("/a", a) + zeroZero.InsertIntoValuesDimension("/a", a) ab := &testValue{ID: "/a/b"} - zeroZero.Insert("/a/b", ab) + zeroZero.InsertIntoValuesDimension("/a/b", ab) - c.Assert(zeroZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 0, Role: 0}) + c.Assert(zeroZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 0}) s, v := zeroZero.LongestPrefix("/a/b/c", true, nil) c.Assert(v, eq, ab) c.Assert(s, eq, "/a/b") // Change language. oneZero := zeroZero.Increment(0) - c.Assert(zeroZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 0, Role: 0}) - c.Assert(oneZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 1, Role: 0}) - - // Change role. - oneOne := oneZero.Increment(1) - c.Assert(zeroZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 0, Role: 0}) - c.Assert(oneZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 1, Role: 0}) - c.Assert(oneOne.Get("/a"), eq, &testValue{ID: "/a", Lang: 1, Role: 1}) + c.Assert(zeroZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 0}) + c.Assert(oneZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 1}) } func TestTreeData(t *testing.T) { @@ -74,17 +67,16 @@ func TestTreeData(t *testing.T) { tree := doctree.New( doctree.Config[*testValue]{ - MatrixRow: []int{0, 0}, - Shifter: &testShifter{}, + Shifter: &testShifter{}, }, ) - tree.Insert("", &testValue{ID: "HOME"}) - tree.Insert("/a", &testValue{ID: "/a"}) - tree.Insert("/a/b", &testValue{ID: "/a/b"}) - tree.Insert("/b", &testValue{ID: "/b"}) - tree.Insert("/b/c", &testValue{ID: "/b/c"}) - tree.Insert("/b/c/d", &testValue{ID: "/b/c/d"}) + tree.InsertIntoValuesDimension("", &testValue{ID: "HOME"}) + tree.InsertIntoValuesDimension("/a", &testValue{ID: "/a"}) + tree.InsertIntoValuesDimension("/a/b", &testValue{ID: "/a/b"}) + tree.InsertIntoValuesDimension("/b", &testValue{ID: "/b"}) + tree.InsertIntoValuesDimension("/b/c", &testValue{ID: "/b/c"}) + tree.InsertIntoValuesDimension("/b/c/d", &testValue{ID: "/b/c/d"}) var values []string @@ -93,7 +85,7 @@ func TestTreeData(t *testing.T) { w := &doctree.NodeShiftTreeWalker[*testValue]{ Tree: tree, WalkContext: ctx, - Handle: func(s string, t *testValue) (bool, error) { + Handle: func(s string, t *testValue, match doctree.DimensionFlag) (bool, error) { ctx.Data().Insert(s, map[string]any{ "id": t.ID, }) @@ -116,27 +108,26 @@ func TestTreeEvents(t *testing.T) { tree := doctree.New( doctree.Config[*testValue]{ - MatrixRow: []int{0, 0}, - Shifter: &testShifter{echo: true}, + Shifter: &testShifter{echo: true}, }, ) - tree.Insert("/a", &testValue{ID: "/a", Weight: 2, IsBranch: true}) - tree.Insert("/a/p1", &testValue{ID: "/a/p1", Weight: 5}) - tree.Insert("/a/p", &testValue{ID: "/a/p2", Weight: 6}) - tree.Insert("/a/s1", &testValue{ID: "/a/s1", Weight: 5, IsBranch: true}) - tree.Insert("/a/s1/p1", &testValue{ID: "/a/s1/p1", Weight: 8}) - tree.Insert("/a/s1/p1", &testValue{ID: "/a/s1/p2", Weight: 9}) - tree.Insert("/a/s1/s2", &testValue{ID: "/a/s1/s2", Weight: 6, IsBranch: true}) - tree.Insert("/a/s1/s2/p1", &testValue{ID: "/a/s1/s2/p1", Weight: 8}) - tree.Insert("/a/s1/s2/p2", &testValue{ID: "/a/s1/s2/p2", Weight: 7}) + tree.InsertIntoValuesDimension("/a", &testValue{ID: "/a", Weight: 2, IsBranch: true}) + tree.InsertIntoValuesDimension("/a/p1", &testValue{ID: "/a/p1", Weight: 5}) + tree.InsertIntoValuesDimension("/a/p", &testValue{ID: "/a/p2", Weight: 6}) + tree.InsertIntoValuesDimension("/a/s1", &testValue{ID: "/a/s1", Weight: 5, IsBranch: true}) + tree.InsertIntoValuesDimension("/a/s1/p1", &testValue{ID: "/a/s1/p1", Weight: 8}) + tree.InsertIntoValuesDimension("/a/s1/p1", &testValue{ID: "/a/s1/p2", Weight: 9}) + tree.InsertIntoValuesDimension("/a/s1/s2", &testValue{ID: "/a/s1/s2", Weight: 6, IsBranch: true}) + tree.InsertIntoValuesDimension("/a/s1/s2/p1", &testValue{ID: "/a/s1/s2/p1", Weight: 8}) + tree.InsertIntoValuesDimension("/a/s1/s2/p2", &testValue{ID: "/a/s1/s2/p2", Weight: 7}) w := &doctree.NodeShiftTreeWalker[*testValue]{ Tree: tree, WalkContext: &doctree.WalkContext[*testValue]{}, } - w.Handle = func(s string, t *testValue) (bool, error) { + w.Handle = func(s string, t *testValue, match doctree.DimensionFlag) (bool, error) { if t.IsBranch { w.WalkContext.AddEventListener("weight", s, func(e *doctree.Event[*testValue]) { if e.Source.Weight > t.Weight { @@ -169,21 +160,20 @@ func TestTreeInsert(t *testing.T) { tree := doctree.New( doctree.Config[*testValue]{ - MatrixRow: []int{0, 0}, - Shifter: &testShifter{}, + Shifter: &testShifter{}, }, ) a := &testValue{ID: "/a"} - tree.Insert("/a", a) + tree.InsertIntoValuesDimension("/a", a) ab := &testValue{ID: "/a/b"} - tree.Insert("/a/b", ab) + tree.InsertIntoValuesDimension("/a/b", ab) - c.Assert(tree.Get("/a"), eq, &testValue{ID: "/a", Lang: 0, Role: 0}) + c.Assert(tree.Get("/a"), eq, &testValue{ID: "/a", Lang: 0}) c.Assert(tree.Get("/notfound"), qt.IsNil) ab2 := &testValue{ID: "/a/b", Lang: 0} - v, ok := tree.Insert("/a/b", ab2) + v, ok := tree.InsertIntoValuesDimension("/a/b", ab2) c.Assert(ok, qt.IsTrue) c.Assert(v, qt.DeepEquals, ab2) @@ -199,8 +189,7 @@ func TestTreePara(t *testing.T) { tree := doctree.New( doctree.Config[*testValue]{ - MatrixRow: []int{0, 0}, - Shifter: &testShifter{}, + Shifter: &testShifter{}, }, ) @@ -210,13 +199,13 @@ func TestTreePara(t *testing.T) { a := &testValue{ID: "/a"} lock := tree.Lock(true) defer lock() - tree.Insert("/a", a) + tree.InsertIntoValuesDimension("/a", a) ab := &testValue{ID: "/a/b"} - tree.Insert("/a/b", ab) + tree.InsertIntoValuesDimension("/a/b", ab) key := fmt.Sprintf("/a/b/c/%d", i) val := &testValue{ID: key} - tree.Insert(key, val) + tree.InsertIntoValuesDimension(key, val) c.Assert(tree.Get(key), eq, val) // s, _ := tree.LongestPrefix(key, nil) // c.Assert(s, eq, "/a/b") @@ -239,6 +228,10 @@ func TestValidateKey(t *testing.T) { c.Assert(doctree.ValidateKey("/abc/"), qt.IsNotNil) } +const ( + testDimension1 doctree.DimensionFlag = 1 << iota +) + type testShifter struct { echo bool } @@ -247,31 +240,31 @@ func (s *testShifter) Dimension(n *testValue, d int) []*testValue { return []*testValue{n} } -func (s *testShifter) Insert(old, new *testValue) (*testValue, bool) { - return new, true +func (s *testShifter) Insert(old, new *testValue) *testValue { + return new } -func (s *testShifter) Delete(n *testValue, dimension []int) (bool, bool) { +func (s *testShifter) InsertInto(old, new *testValue, dimension doctree.Dimension) *testValue { + return new +} + +func (s *testShifter) Delete(n *testValue, dimension doctree.Dimension) (bool, bool) { return true, true } -func (s *testShifter) Shift(n *testValue, dimension []int, exact bool) (*testValue, bool) { +func (s *testShifter) Shift(n *testValue, dimension doctree.Dimension, exact bool) (*testValue, bool, doctree.DimensionFlag) { if s.echo { - return n, true + return n, true, testDimension1 } if n.NoCopy { - if n.Lang == dimension[0] && n.Role == dimension[1] { - return n, true + if n.Lang == dimension[0] { + return n, true, testDimension1 } - return nil, false - } - if len(dimension) != 2 { - panic("invalid dimension") + return nil, false, testDimension1 } c := *n c.Lang = dimension[0] - c.Role = dimension[1] - return &c, true + return &c, true, testDimension1 } func (s *testShifter) All(n *testValue) []*testValue { @@ -281,7 +274,6 @@ func (s *testShifter) All(n *testValue) []*testValue { type testValue struct { ID string Lang int - Role int Weight int IsBranch bool @@ -294,14 +286,13 @@ func BenchmarkTreeInsert(b *testing.B) { for i := 0; i < b.N; i++ { tree := doctree.New( doctree.Config[*testValue]{ - MatrixRow: []int{0, 0}, - Shifter: &testShifter{}, + Shifter: &testShifter{}, }, ) for i := 0; i < numElements; i++ { - lang, role := rand.Intn(2), rand.Intn(2) - tree.Insert(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Role: role, Weight: i, NoCopy: true}) + lang := rand.Intn(2) + tree.InsertIntoValuesDimension(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Weight: i, NoCopy: true}) } } } @@ -329,20 +320,19 @@ func BenchmarkWalk(b *testing.B) { createTree := func() *doctree.NodeShiftTree[*testValue] { tree := doctree.New( doctree.Config[*testValue]{ - MatrixRow: []int{0, 0}, - Shifter: &testShifter{}, + Shifter: &testShifter{}, }, ) for i := 0; i < numElements; i++ { - lang, role := rand.Intn(2), rand.Intn(2) - tree.Insert(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Role: role, Weight: i, NoCopy: true}) + lang := rand.Intn(2) + tree.InsertIntoValuesDimension(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Weight: i, NoCopy: true}) } return tree } - handle := func(s string, t *testValue) (bool, error) { + handle := func(s string, t *testValue, match doctree.DimensionFlag) (bool, error) { return false, nil } @@ -366,7 +356,7 @@ func BenchmarkWalk(b *testing.B) { base := createTree() b.ResetTimer() for i := 0; i < b.N; i++ { - for d1 := 0; d1 < 2; d1++ { + for d1 := 0; d1 < 1; d1++ { for d2 := 0; d2 < 2; d2++ { tree := base.Shape(d1, d2) w := &doctree.NodeShiftTreeWalker[*testValue]{ diff --git a/hugolib/doctree/nodeshifttree.go b/hugolib/doctree/nodeshifttree.go index e5856578bc9..6c3b1161ce4 100644 --- a/hugolib/doctree/nodeshifttree.go +++ b/hugolib/doctree/nodeshifttree.go @@ -26,10 +26,6 @@ import ( type ( Config[T any] struct { - // MatrixRow configures a row in the matrix (e.g. [role, language]). - // It cannot be changed once set. - MatrixRow Dimensions - // Shifter handles tree transformations. Shifter Shifter[T] } @@ -39,18 +35,23 @@ type ( // Dimension gets all values of node n in dimension d. Dimension(n T, d int) []T - // Insert inserts new into the correct dimension. + // Insert inserts new into the tree into the dimension it provides. + // It may replace old. + // It returns a T (can be the same as old). + Insert(old, new T) T + + // Insert inserts new into the given dimension. // It may replace old. - // It returns a T (can be the same as old) and a bool indicating if the insert was successful. - Insert(old, new T) (T, bool) + // It returns a T (can be the same as old). + InsertInto(old, new T, dimension Dimension) T // Delete deletes T from the given dimension and returns whether the dimension was deleted and if it's empty after the delete. - Delete(v T, dimension []int) (bool, bool) + Delete(v T, dimension Dimension) (bool, bool) - // Shift shifts T into the given dimension. - // It may return a zero value and false. - // If exact is set, it will only return an exact match. - Shift(v T, dimension []int, exact bool) (T, bool) + // Shift shifts T into the given dimension + // and returns the shifted T and a bool indicating if the shift was successful and + // how accurate a match T is according to its dimensions. + Shift(v T, dimension Dimension, exact bool) (T, bool, DimensionFlag) // All returns all values of T in all dimensions. All(n T) []T @@ -64,7 +65,7 @@ type NodeShiftTree[T any] struct { tree *radix.Tree // E.g. [language, role]. - dims Dimensions + dims Dimension shifter Shifter[T] mu *sync.RWMutex @@ -75,13 +76,8 @@ func New[T any](cfg Config[T]) *NodeShiftTree[T] { panic("Shifter is required") } - if len(cfg.MatrixRow) == 0 { - panic("At least one dimension is required") - } - return &NodeShiftTree[T]{ mu: &sync.RWMutex{}, - dims: cfg.MatrixRow, shifter: cfg.Shifter, tree: radix.New(), } @@ -147,20 +143,20 @@ func (t *NodeShiftTree[T]) Increment(d int) *NodeShiftTree[T] { return t.Shape(d, t.dims[d]+1) } -func (r *NodeShiftTree[T]) Insert(s string, v T) (T, bool) { - s = cleanKey(s) - mustValidateKey(s) - vv, ok := r.tree.Get(s) - - if ok { - v, ok = r.shifter.Insert(vv.(T), v) - if !ok { - return v, false - } +func (r *NodeShiftTree[T]) InsertIntoCurrentDimension(s string, v T) (T, bool) { + s = mustValidateKey(cleanKey(s)) + if vv, ok := r.tree.Get(s); ok { + v = r.shifter.InsertInto(vv.(T), v, r.dims) } + r.tree.Insert(s, v) + return v, true +} - // fmt.Printf("Insert2 %q -> %T\n", s, v) - +func (r *NodeShiftTree[T]) InsertIntoValuesDimension(s string, v T) (T, bool) { + s = mustValidateKey(cleanKey(s)) + if vv, ok := r.tree.Get(s); ok { + v = r.shifter.Insert(vv.(T), v) + } r.tree.Insert(s, v) return v, true } @@ -174,7 +170,7 @@ func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v any) (any, bool) { func (r *NodeShiftTree[T]) InsertWithLock(s string, v T) (T, bool) { r.mu.Lock() defer r.mu.Unlock() - return r.Insert(s, v) + return r.InsertIntoValuesDimension(s, v) } func (t *NodeShiftTree[T]) Len() int { @@ -213,7 +209,7 @@ func (r *NodeShiftTree[T]) LongestPrefix(s string, exact bool, predicate func(v longestPrefix, v, found := r.tree.LongestPrefix(s) if found { - if t, ok := r.shift(v.(T), exact); ok && (predicate == nil || predicate(t)) { + if t, ok, _ := r.shift(v.(T), exact); ok && (predicate == nil || predicate(t)) { return longestPrefix, t } } @@ -294,7 +290,7 @@ type NodeShiftTreeWalker[T any] struct { // Handle will be called for each node in the main tree. // If the callback returns true, the walk will stop. // The callback can optionally return a callback for the nested tree. - Handle func(string, T) (terminate bool, err error) + Handle func(s string, v T, exact DimensionFlag) (terminate bool, err error) // Optional prefix filter. Prefix string @@ -364,13 +360,13 @@ func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error { return false } - t, ok := r.toT(r.Tree, v) + t, ok, exact := r.toT(r.Tree, v) if !ok { return false } var terminate bool - terminate, err = r.Handle(s, t) + terminate, err = r.Handle(s, t, exact) if terminate || err != nil { return true } @@ -394,12 +390,12 @@ func (r *NodeShiftTreeWalker[T]) resetLocalState() { r.skipPrefixes = nil } -func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v any) (t T, ok bool) { +func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v any) (t T, ok bool, exact DimensionFlag) { if r.NoShift { t = v.(T) ok = true } else { - t, ok = tree.shift(v.(T), r.Exact) + t, ok, exact = tree.shift(v.(T), r.Exact) } return } @@ -410,14 +406,10 @@ func (r *NodeShiftTree[T]) Has(s string) bool { } func (t NodeShiftTree[T]) clone() *NodeShiftTree[T] { - dimensions := make(Dimensions, len(t.dims)) - copy(dimensions, t.dims) - t.dims = dimensions - return &t } -func (r *NodeShiftTree[T]) shift(t T, exact bool) (T, bool) { +func (r *NodeShiftTree[T]) shift(t T, exact bool) (T, bool, DimensionFlag) { return r.shifter.Shift(t, r.dims, exact) } @@ -428,7 +420,7 @@ func (r *NodeShiftTree[T]) get(s string) (T, bool) { var t T return t, false } - t, ok := r.shift(v.(T), true) + t, ok, _ := r.shift(v.(T), true) return t, ok } diff --git a/hugolib/doctree/support.go b/hugolib/doctree/support.go index f2e3000ff86..8083df127ac 100644 --- a/hugolib/doctree/support.go +++ b/hugolib/doctree/support.go @@ -243,8 +243,9 @@ func (ctx *WalkContext[T]) HandleEventsAndHooks() error { return nil } -func mustValidateKey(key string) { +func mustValidateKey(key string) string { if err := ValidateKey(key); err != nil { panic(err) } + return key } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 231f808c87c..80e75445374 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -370,7 +370,7 @@ func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) { w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: s.pageMap.treePages, LockType: doctree.LockTypeRead, - Handle: func(s string, n contentNodeI) (bool, error) { + Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { return fn(s, n.(*pageState)), nil }, } diff --git a/hugolib/page.go b/hugolib/page.go index 0384199c879..2bd4c49f190 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -21,6 +21,7 @@ import ( "sync/atomic" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" @@ -359,7 +360,7 @@ func (p *pageState) AllTranslations() page.Pages { page.SortByLanguage(pasc) return pasc, nil } - all := p.s.pageMap.treePages.GetDimension(p.Path(), pageTreeDimensionLanguage) + all := p.s.pageMap.treePages.GetDimension(p.Path(), doctree.DimensionLanguage.Index()) var pas page.Pages for _, p := range all { if p == nil { diff --git a/hugolib/page__tree.go b/hugolib/page__tree.go index 67b082bf04e..e54d596bc95 100644 --- a/hugolib/page__tree.go +++ b/hugolib/page__tree.go @@ -154,7 +154,7 @@ func (pt pageTree) Sections() page.Pages { Tree: tree, Prefix: prefix, } - w.Handle = func(ss string, n contentNodeI) (bool, error) { + w.Handle = func(ss string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if !n.isContentNodeBranch() { return false, nil } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index b764a23f5c9..f5ff95f3c29 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -1396,7 +1396,7 @@ Resources: {{ range .Resources }}{{ .RelPermalink }}|{{ .Content }}|{{ end }}| b.AssertFileContent("public/nn/sect/mybundle_nn/index.html", "TranslationKey: adfasdf|", - "Title: mybundle nn|TranslationKey: adfasdf|\nResources: /en/sect/mybundle_en/f1.txt|f1.en|/nn/sect/mybundle_nn/f2.txt|f2.nn||", + "Title: mybundle nn|TranslationKey: adfasdf|\nResources: /en/sect/mybundle_en/f1.txt|f1.en|/nn/sect/mybundle_nn/f2.nn.txt|f2.nn||", ) } diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index 2b75ddf608c..123d752e0f4 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -174,7 +174,7 @@ Resources: {{ range .Resources }}RelPermalink: {{ .RelPermalink }}|Content: {{ . b := Test(t, files) b.AssertFileContent("public/en/mybundle/index.html", "My Bundle|/en/mybundle/|en|\nResources: RelPermalink: /en/mybundle/f1.txt|Content: F1|RelPermalink: /en/mybundle/f2.txt|Content: F2||") - b.AssertFileContent("public/nn/mybundle/index.html", "My Bundle NN|/nn/mybundle/|nn|\nResources: RelPermalink: /en/mybundle/f1.txt|Content: F1|RelPermalink: /nn/mybundle/f2.txt|Content: F2 nn.||") + b.AssertFileContent("public/nn/mybundle/index.html", "My Bundle NN|/nn/mybundle/|nn|\nResources: RelPermalink: /en/mybundle/f1.txt|Content: F1|RelPermalink: /nn/mybundle/f2.nn.txt|Content: F2 nn.||") } func TestMultilingualDisableLanguage(t *testing.T) { @@ -420,6 +420,77 @@ Single content. b.AssertFileContent("public/section-not-bundle/single/index.html", "Section Single", "|

Single content.

") } +func TestBundledResourcesMultilingualDuplicateResourceFiles(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +baseURL = "https://example.com/" +[markup] +[markup.goldmark] +duplicateResourceFiles = true +[languages] +[languages.en] +weight = 1 +[languages.en.permalinks] +"/" = "/enpages/:slug/" +[languages.nn] +weight = 2 +[languages.nn.permalinks] +"/" = "/nnpages/:slug/" +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +{{< getresource "f1.txt" >}} +{{< getresource "f2.txt" >}} +-- content/mybundle/index.nn.md -- +--- +title: "My Bundle NN" +--- +{{< getresource "f1.txt" >}} +f2.nn.txt is the original name. +{{< getresource "f2.nn.txt" >}} +{{< getresource "f2.txt" >}} +{{< getresource "sub/f3.txt" >}} +-- content/mybundle/f1.txt -- +F1 en. +-- content/mybundle/sub/f3.txt -- +F1 en. +-- content/mybundle/f2.txt -- +F2 en. +-- content/mybundle/f2.nn.txt -- +F2 nn. +-- layouts/shortcodes/getresource.html -- +{{ $r := .Page.Resources.Get (.Get 0)}} +Resource: {{ (.Get 0) }}|{{ with $r }}{{ .RelPermalink }}|{{ .Content }}|{{ else }}Not found.{{ end}} +-- layouts/_default/single.html -- +{{ .Title }}|{{ .RelPermalink }}|{{ .Lang }}|{{ .Content }}| +` + b := Test(t, files) + + // helpers.PrintFs(b.H.Fs.PublishDir, "", os.Stdout) + b.AssertFileContent("public/nn/nnpages/my-bundle-nn/index.html", ` +My Bundle NN +Resource: f1.txt|/nn/nnpages/my-bundle-nn/f1.txt| +Resource: f2.txt|/nn/nnpages/my-bundle-nn/f2.nn.txt|F2 nn.| +Resource: f2.nn.txt|/nn/nnpages/my-bundle-nn/f2.nn.txt|F2 nn.| +Resource: sub/f3.txt|/nn/nnpages/my-bundle-nn/sub/f3.txt|F1 en.| +`) + + b.AssertFileContent("public/enpages/my-bundle/f2.txt", "F2 en.") + b.AssertFileContent("public/nn/nnpages/my-bundle-nn/f2.nn.txt", "F2 nn") + + b.AssertFileContent("public/enpages/my-bundle/index.html", ` +Resource: f1.txt|/enpages/my-bundle/f1.txt|F1 en.| +Resource: f2.txt|/enpages/my-bundle/f2.txt|F2 en.| +`) + b.AssertFileContent("public/enpages/my-bundle/f1.txt", "F1 en.") + + // Should be duplicated to the nn bundle. + b.AssertFileContent("public/nn/nnpages/my-bundle-nn/f1.txt", "F1 en.") +} + // https://github.com/gohugoio/hugo/issues/5858 func TestBundledResourcesWhenMultipleOutputFormats(t *testing.T) { t.Parallel() diff --git a/hugolib/site.go b/hugolib/site.go index 972cfeb19d3..312f6b97f1c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -213,7 +213,7 @@ func (s *Site) initRenderFormats() { w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: s.pageMap.treePages, - Handle: func(key string, n contentNodeI) (bool, error) { + Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if p, ok := n.(*pageState); ok { for _, f := range p.m.configuredOutputFormats { if !formatSet[f.Name] { diff --git a/hugolib/site_new.go b/hugolib/site_new.go index f44501c9e07..5fc4c9cf3d9 100644 --- a/hugolib/site_new.go +++ b/hugolib/site_new.go @@ -146,32 +146,22 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { confm := cfg.Configs var sites []*Site - // Set up the page trees. - matrixRow := []int{ - // language - 0, + ns := &contentNodeShifter{ + numLanguages: len(confm.Languages), } - ns := &contentNodeShifter{} - - pageTreeConfig := doctree.Config[contentNodeI]{ - MatrixRow: matrixRow, - Shifter: ns, - } - - resourceTreeConfig := doctree.Config[contentNodeI]{ - MatrixRow: matrixRow, - Shifter: ns, + treeConfig := doctree.Config[contentNodeI]{ + Shifter: ns, } pageTrees := &pageTrees{ treePages: doctree.New( - pageTreeConfig, + treeConfig, ), treeResources: doctree.New( - resourceTreeConfig, + treeConfig, ), - treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](pageTreeDimensionLanguage, len(confm.Languages)), + treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](doctree.DimensionLanguage.Index(), len(confm.Languages)), } pageTrees.treePagesResources = doctree.WalkableTrees[contentNodeI]{ diff --git a/hugolib/site_render.go b/hugolib/site_render.go index f9e9125f310..379dd6e867b 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -83,7 +83,7 @@ func (s *Site) renderPages(ctx *siteRenderContext) error { w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: s.pageMap.treePages, - Handle: func(key string, n contentNodeI) (bool, error) { + Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { if p, ok := n.(*pageState); ok { if cfg.shouldRender(p) { select { @@ -266,7 +266,7 @@ func (s *Site) renderPaginator(p *pageState, templ tpl.Template) error { func (s *Site) renderAliases() error { w := &doctree.NodeShiftTreeWalker[contentNodeI]{ Tree: s.pageMap.treePages, - Handle: func(key string, n contentNodeI) (bool, error) { + Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { p := n.(*pageState) // We cannot alias a page that's not rendered. diff --git a/markup/goldmark/goldmark_config/config.go b/markup/goldmark/goldmark_config/config.go index cdfb4e7cc00..739913f69b6 100644 --- a/markup/goldmark/goldmark_config/config.go +++ b/markup/goldmark/goldmark_config/config.go @@ -66,9 +66,10 @@ var Default = Config{ // Config configures Goldmark. type Config struct { - Renderer Renderer - Parser Parser - Extensions Extensions + DuplicateResourceFiles bool + Renderer Renderer + Parser Parser + Extensions Extensions } type Extensions struct { diff --git a/markup/markup.go b/markup/markup.go index ebd86f38fc0..835c7bbecac 100644 --- a/markup/markup.go +++ b/markup/markup.go @@ -95,6 +95,7 @@ func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, erro type ConverterProvider interface { Get(name string) converter.Provider + IsGoldmark(name string) bool // Default() converter.Provider GetMarkupConfig() markup_config.Config GetHighlighter() highlight.Highlighter @@ -110,6 +111,11 @@ type converterRegistry struct { config converter.ProviderConfig } +func (r *converterRegistry) IsGoldmark(name string) bool { + cp := r.Get(name) + return cp != nil && cp.Name() == "goldmark" +} + func (r *converterRegistry) Get(name string) converter.Provider { return r.converters[strings.ToLower(name)] } diff --git a/resources/resource.go b/resources/resource.go index 1c5fd0f47e0..29f9f81720d 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -62,6 +62,9 @@ type ResourceSourceDescriptor struct { // The name of the resource. Name string + // The name of the resource as it was read from the source. + NameOriginal string + // Any base paths prepended to the target path. This will also typically be the // language code, but setting it here means that it should not have any effect on // the permalink. @@ -136,6 +139,10 @@ func (fd *ResourceSourceDescriptor) init(r *Spec) error { fd.Name = fd.TargetPath } + if fd.NameOriginal == "" { + fd.NameOriginal = fd.Name + } + mediaType := fd.MediaType if mediaType.IsZero() { ext := fd.Path.Ext() @@ -447,6 +454,10 @@ func (l *genericResource) Name() string { return l.name } +func (l *genericResource) NameOriginal() string { + return l.sd.NameOriginal +} + func (l *genericResource) Params() maps.Params { return l.params } diff --git a/resources/resource/resources.go b/resources/resource/resources.go index 2f9ee332a2e..9f298b7a6c6 100644 --- a/resources/resource/resources.go +++ b/resources/resource/resources.go @@ -55,16 +55,33 @@ func (r Resources) ByType(typ any) Resources { // Get locates the name given in Resources. // The search is case insensitive. func (r Resources) Get(name any) Resource { + if r == nil { + return nil + } namestr, err := cast.ToStringE(name) if err != nil { panic(err) } namestr = strings.ToLower(namestr) + + // First check the Name. + // Note that this can be modified by the user in the front matter, + // also, it does not contain any language code. for _, resource := range r { if strings.EqualFold(namestr, resource.Name()) { return resource } } + + // Finally, check the original name. + for _, resource := range r { + if nop, ok := resource.(NameOriginalProvider); ok { + if strings.EqualFold(namestr, nop.NameOriginal()) { + return resource + } + } + } + return nil } diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 4d20c89fb4a..43d0aa7864e 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -126,12 +126,22 @@ type ResourceNameTitleProvider interface { // So, for the image "/some/path/sunset.jpg" this will be "sunset.jpg". // The value returned by this method will be used in the GetByPrefix and ByPrefix methods // on Resources. + // Note that for bundled content resources with language code in the filename, this will + // be the name without the language code. Name() string // Title returns the title if set in front matter. For content pages, this will be the expected value. Title() string } +type NameOriginalProvider interface { + // NameOriginal is the original name of this resource. + // Note that for bundled content resources with language code in the filename, this will + // be the name with the language code. + // For internal use (for now). + NameOriginal() string +} + type ResourceParamsProvider interface { // Params set in front matter for this resource. Params() maps.Params diff --git a/resources/resource_metadata.go b/resources/resource_metadata.go index 6bc31084f06..869fc11bf5e 100644 --- a/resources/resource_metadata.go +++ b/resources/resource_metadata.go @@ -28,9 +28,10 @@ import ( ) var ( - _ mediaTypeAssigner = (*genericResource)(nil) - _ mediaTypeAssigner = (*imageResource)(nil) - _ resource.Staler = (*genericResource)(nil) + _ mediaTypeAssigner = (*genericResource)(nil) + _ mediaTypeAssigner = (*imageResource)(nil) + _ resource.Staler = (*genericResource)(nil) + _ resource.NameOriginalProvider = (*genericResource)(nil) ) // metaAssigner allows updating metadata in resources that supports it. diff --git a/resources/transform.go b/resources/transform.go index c0ae19e8649..57c5794ff82 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -55,6 +55,7 @@ var ( _ resource.WithResourceMetaProvider = (*resourceAdapter)(nil) _ identity.DependencyManagerProvider = (*resourceAdapter)(nil) _ identity.IdentityGroupProvider = (*resourceAdapter)(nil) + _ resource.NameOriginalProvider = (*resourceAdapter)(nil) ) // These are transformations that need special support in Hugo that may not @@ -279,6 +280,11 @@ func (r *resourceAdapter) Name() string { return r.metaProvider.Name() } +func (r *resourceAdapter) NameOriginal() string { + r.init(false, false) + return r.target.(resource.NameOriginalProvider).NameOriginal() +} + func (r *resourceAdapter) Params() maps.Params { r.init(false, false) return r.metaProvider.Params()