diff --git a/cache/dynacache/dynacache.go b/cache/dynacache/dynacache.go index 85b36013867..dd4c0473c06 100644 --- a/cache/dynacache/dynacache.go +++ b/cache/dynacache/dynacache.go @@ -156,12 +156,15 @@ func (c *Cache) ClearMatching(predicate func(k, v any) bool) { g.Wait() } -// ClearOnRebuild prepares the cache for a new rebuild taking the given changeset into account. -func (c *Cache) ClearOnRebuild(changeset ...identity.Identity) { +// ClearOnRebuild prepares the cache for a new rebuild taking the given predicate and changeset into account. +func (c *Cache) ClearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity) { + if predicate == nil { + predicate = func(k, v any) bool { return false } + } g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{ NumWorkers: len(c.partitions), Handle: func(ctx context.Context, partition PartitionManager) error { - partition.clearOnRebuild(changeset...) + partition.clearOnRebuild(predicate, changeset...) return nil }, }) @@ -420,7 +423,7 @@ func (p *Partition[K, V]) clearMatching(predicate func(k, v any) bool) { }) } -func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) { +func (p *Partition[K, V]) clearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity) { opts := p.getOptions() if opts.ClearWhen == ClearNever { return @@ -440,6 +443,10 @@ func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) { return true } + if predicate(key, v) { + return true + } + // Now check if this entry has changed based on the changeset // based on filesystem events. if len(changeset) == 0 { @@ -542,7 +549,7 @@ type PartitionManager interface { adjustMaxSize(addend int) int getMaxSize() int getOptions() OptionsPartition - clearOnRebuild(changeset ...identity.Identity) + clearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity) clearMatching(predicate func(k, v any) bool) clearStale() } diff --git a/cache/dynacache/dynacache_test.go b/cache/dynacache/dynacache_test.go index 53de2385e84..82f56b6f149 100644 --- a/cache/dynacache/dynacache_test.go +++ b/cache/dynacache/dynacache_test.go @@ -144,13 +144,13 @@ func TestClear(t *testing.T) { c.Assert(cache.Keys(predicateAll), qt.HasLen, 4) - cache.ClearOnRebuild() + cache.ClearOnRebuild(nil) // Stale items are always cleared. c.Assert(cache.Keys(predicateAll), qt.HasLen, 2) cache = newTestCache(t) - cache.ClearOnRebuild(identity.StringIdentity("changed")) + cache.ClearOnRebuild(nil, identity.StringIdentity("changed")) c.Assert(cache.Keys(nil), qt.HasLen, 1) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index 6273870b73f..4677b491424 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -1018,31 +1018,27 @@ func (h *HugoSites) resolveAndClearStateForIdentities( } // The order matters here: - // 1. Handle the cache busters first, as those may produce identities for the page reset step. + // 1. Clear the cache first; this may produce identities that need to be resolved. // 2. Then reset the page outputs, which may mark some resources as stale. - // 3. Then GC the cache. - if cachebuster != nil { - if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { - ll := l.WithField("substep", "gc dynacache cachebuster") + // 3. Then clear the cache for stale resources. + if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { + ll := l.WithField("substep", "gc dynacache step 1") - shouldDelete := func(k, v any) bool { - if cachebuster == nil { - return false + shouldDelete := func(k, v any) bool { + if cachebuster != nil { + if s, ok := k.(string); ok && cachebuster(s) { + return true } - var b bool - if s, ok := k.(string); ok { - b = cachebuster(s) - } - - return b } - h.MemCache.ClearMatching(shouldDelete) - - return ll, nil - }); err != nil { - return err + return false } + + h.MemCache.ClearOnRebuild(shouldDelete, changes...) + + return ll, nil + }); err != nil { + return err } // Drain the the cache eviction stack. @@ -1079,9 +1075,15 @@ func (h *HugoSites) resolveAndClearStateForIdentities( } if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { - ll := l.WithField("substep", "gc dynacache") + ll := l.WithField("substep", "gc dynacache step 2") + + // We need to do this twice to catch all the resources that are marked as stale in the first pass. + for i := 0; i < 2; i++ { + h.MemCache.ClearMatching(func(k, v any) bool { + return resource.IsStaleAny(v) + }) + } - h.MemCache.ClearOnRebuild(changes...) h.Log.Trace(logg.StringFunc(func() string { var sb strings.Builder sb.WriteString("dynacache keys:\n") diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go index b1ebe14d53a..9d082c9b79d 100644 --- a/hugolib/rebuild_test.go +++ b/hugolib/rebuild_test.go @@ -1261,3 +1261,65 @@ func BenchmarkRebuildContentFileChange(b *testing.B) { // fmt.Println(bb.LogString()) } } + +func TestRebuildConcat(t *testing.T) { + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +disableLiveReload = true +-- assets/a.css -- +a +-- assets/b.css -- +b +-- assets/c.css -- +c +-- layouts/index.html -- +{{ $a := resources.GetMatch "a.css" }} +{{ $b := resources.Get "b.css" }} +{{ $c := resources.GetMatch "c.css" }} +{{ $bc := slice $b $c | resources.Concat "bc.css" }} +{{ $abbc := slice $a $b $bc | resources.Concat "abbc.css" }} +abbc: {{ $abbc.Content | safeCSS }} +` + b := TestRunning(t, files) + + b.AssertFileContent("public/index.html", "abbc: abbc") + + b.EditFileReplaceAll("assets/a.css", "a", "a edited").Build() + + b.AssertFileContent("public/index.html", "abbc: a editedbbc") + + b.EditFileReplaceAll("assets/c.css", "c", "c edited").Build() + + b.AssertFileContent("public/index.html", "abbc: a editedbbc edited") +} + +func TestRebuildConcatMatch(t *testing.T) { + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +disableLiveReload = true +disableKinds = ["taxonomy", "term", "sitemap", "robotsTXT", "404", "rss"] +-- assets/a.css -- +a +-- assets/b.css -- +b +-- assets/c.css -- +c +-- layouts/baseof.html -- +{{ block "main" . }}{{ end }} +{{ $concat := resources.Match "*.css" | resources.Concat "all.css" }} +{{ $a := resources.Get "a.css" }} +{{ $all := slice $a $concat | resources.Concat "all.css" | fingerprint }} +all: {{ $all.RelPermalink }} +-- layouts/index.html -- +{{ define "main" }}Main{{ end }} + +` + b := TestRunning(t, files) + + b.AssertFileContent("public/index.html", "all: /all.ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad.css") + b.EditFileReplaceAll("assets/c.css", "c", "c edited").Build() + b.AssertFileContent("public/index.html", "all: /all.6dc8c9da5de8f1a85a6167148657c09f26be309eaa9527d757731f91e57e59e5.css") + b.AssertFileContent("public/all.6dc8c9da5de8f1a85a6167148657c09f26be309eaa9527d757731f91e57e59e5.css", "abc edited") +} diff --git a/identity/identity.go b/identity/identity.go index ccb2f6e79d3..2720edbe321 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -312,7 +312,7 @@ func (im *identityManager) AddIdentity(ids ...Identity) { im.mu.Lock() for _, id := range ids { - if id == Anonymous { + if id == nil || id == Anonymous { continue } if _, found := im.ids[id]; !found { @@ -420,6 +420,9 @@ func walkIdentitiesShallow(v any, level int, cb func(level int, id Identity) boo if id == Anonymous { return false } + if id == nil { + return false + } return cb(level, id) } diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index c255da601ae..a62d139a3c5 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -20,6 +20,7 @@ import ( "path" "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" @@ -82,6 +83,11 @@ func (r *multiReadSeekCloser) Close() error { func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resource, error) { targetPath = path.Clean(targetPath) return c.rs.ResourceCache.GetOrCreate(targetPath, func() (resource.Resource, error) { + var idm identity.Manager = identity.NopManager + if c.rs.Cfg.Watching() { + idm = identity.NewManager("concat") + } + var resolvedm media.Type // The given set of resources must be of the same Media Type. @@ -91,6 +97,10 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou return nil, fmt.Errorf("resources in Concat must be of the same Media Type, got %q and %q", r.MediaType().Type, resolvedm.Type) } resolvedm = r.MediaType() + identity.WalkIdentitiesShallow(r, func(depth int, id identity.Identity) bool { + idm.AddIdentity(id) + return false + }) } concatr := func() (hugio.ReadSeekCloser, error) { @@ -136,6 +146,7 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou LazyPublish: true, OpenReadSeekCloser: concatr, TargetPath: targetPath, + DependencyManager: idm, }) if err != nil { return nil, err diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index e98eb742599..7e8eab0743b 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -64,7 +64,7 @@ func (c *Client) Copy(r resource.Resource, targetPath string) (resource.Resource } func (c *Client) newDependencyManager() identity.Manager { - if c.rs.Cfg.Running() { + if c.rs.Cfg.Watching() { return identity.NewManager("resources") } return identity.NopManager