Fix rebuild with resources.Concat

Fixes #12017

cache/dynacache/dynacache.go

@@ -156,12 +156,15 @@ func (c *Cache) ClearMatching(predicate func(k, v any) bool) {
 	g.Wait()
 }
 
-// ClearOnRebuild prepares the cache for a new rebuild taking the given changeset into account.
-func (c *Cache) ClearOnRebuild(changeset ...identity.Identity) {
+// ClearOnRebuild prepares the cache for a new rebuild taking the given predicate and changeset into account.
+func (c *Cache) ClearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity) {
+	if predicate == nil {
+		predicate = func(k, v any) bool { return false }
+	}
 	g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
 		NumWorkers: len(c.partitions),
 		Handle: func(ctx context.Context, partition PartitionManager) error {
-			partition.clearOnRebuild(changeset...)
+			partition.clearOnRebuild(predicate, changeset...)
 			return nil
 		},
 	})
@@ -420,7 +423,7 @@ func (p *Partition[K, V]) clearMatching(predicate func(k, v any) bool) {
 	})
 }
 
-func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
+func (p *Partition[K, V]) clearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity) {
 	opts := p.getOptions()
 	if opts.ClearWhen == ClearNever {
 		return
@@ -440,6 +443,10 @@ func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
 			return true
 		}
 
+		if predicate(key, v) {
+			return true
+		}
+
 		// Now check if this entry has changed based on the changeset
 		// based on filesystem events.
 		if len(changeset) == 0 {
@@ -542,7 +549,7 @@ type PartitionManager interface {
 	adjustMaxSize(addend int) int
 	getMaxSize() int
 	getOptions() OptionsPartition
-	clearOnRebuild(changeset ...identity.Identity)
+	clearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity)
 	clearMatching(predicate func(k, v any) bool)
 	clearStale()
 }

cache/dynacache/dynacache_test.go

@@ -144,13 +144,13 @@ func TestClear(t *testing.T) {
 
 	c.Assert(cache.Keys(predicateAll), qt.HasLen, 4)
 
-	cache.ClearOnRebuild()
+	cache.ClearOnRebuild(nil)
 
 	// Stale items are always cleared.
 	c.Assert(cache.Keys(predicateAll), qt.HasLen, 2)
 
 	cache = newTestCache(t)
-	cache.ClearOnRebuild(identity.StringIdentity("changed"))
+	cache.ClearOnRebuild(nil, identity.StringIdentity("changed"))
 
 	c.Assert(cache.Keys(nil), qt.HasLen, 1)
 

hugolib/content_map_page.go

@@ -1018,31 +1018,27 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
 	}
 
 	// The order matters here:
-	// 1. Handle the cache busters first, as those may produce identities for the page reset step.
+	// 1. Clear the cache first; this may produce identities that need to be resolved.
 	// 2. Then reset the page outputs, which may mark some resources as stale.
-	// 3. Then GC the cache.
-	if cachebuster != nil {
-		if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
-			ll := l.WithField("substep", "gc dynacache cachebuster")
+	// 3. Then clear the cache for stale resources.
+	if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
+		ll := l.WithField("substep", "gc dynacache step 1")
 
-			shouldDelete := func(k, v any) bool {
-				if cachebuster == nil {
-					return false
+		shouldDelete := func(k, v any) bool {
+			if cachebuster != nil {
+				if s, ok := k.(string); ok && cachebuster(s) {
+					return true
 				}
-				var b bool
-				if s, ok := k.(string); ok {
-					b = cachebuster(s)
-				}
-
-				return b
 			}
 
-			h.MemCache.ClearMatching(shouldDelete)
-
-			return ll, nil
-		}); err != nil {
-			return err
+			return false
 		}
+
+		h.MemCache.ClearOnRebuild(shouldDelete, changes...)
+
+		return ll, nil
+	}); err != nil {
+		return err
 	}
 
 	// Drain the the cache eviction stack.
@@ -1079,9 +1075,15 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
 	}
 
 	if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
-		ll := l.WithField("substep", "gc dynacache")
+		ll := l.WithField("substep", "gc dynacache step 2")
+
+		// We need to do this twice to catch all the resources that are marked as stale in the first pass.
+		for i := 0; i < 2; i++ {
+			h.MemCache.ClearMatching(func(k, v any) bool {
+				return resource.IsStaleAny(v)
+			})
+		}
 
-		h.MemCache.ClearOnRebuild(changes...)
 		h.Log.Trace(logg.StringFunc(func() string {
 			var sb strings.Builder
 			sb.WriteString("dynacache keys:\n")

hugolib/rebuild_test.go

@@ -1261,3 +1261,65 @@ func BenchmarkRebuildContentFileChange(b *testing.B) {
 		// fmt.Println(bb.LogString())
 	}
 }
+
+func TestRebuildConcat(t *testing.T) {
+	files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+disableLiveReload = true
+-- assets/a.css --
+a
+-- assets/b.css --
+b
+-- assets/c.css --
+c
+-- layouts/index.html --
+{{ $a := resources.GetMatch "a.css" }}
+{{ $b := resources.Get "b.css" }}
+{{ $c := resources.GetMatch "c.css" }}
+{{ $bc := slice $b $c | resources.Concat "bc.css" }}
+{{ $abbc := slice $a $b $bc | resources.Concat "abbc.css" }}
+abbc: {{ $abbc.Content | safeCSS }}
+`
+	b := TestRunning(t, files)
+
+	b.AssertFileContent("public/index.html", "abbc: abbc")
+
+	b.EditFileReplaceAll("assets/a.css", "a", "a edited").Build()
+
+	b.AssertFileContent("public/index.html", "abbc: a editedbbc")
+
+	b.EditFileReplaceAll("assets/c.css", "c", "c edited").Build()
+
+	b.AssertFileContent("public/index.html", "abbc: a editedbbc edited")
+}
+
+func TestRebuildConcatMatch(t *testing.T) {
+	files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+disableLiveReload = true
+disableKinds = ["taxonomy", "term", "sitemap", "robotsTXT", "404", "rss"]
+-- assets/a.css --
+a
+-- assets/b.css --
+b
+-- assets/c.css --
+c
+-- layouts/baseof.html --
+{{ block "main" . }}{{ end }}
+{{ $concat := resources.Match "*.css" | resources.Concat "all.css" }}
+{{ $a := resources.Get "a.css" }}
+{{ $all := slice $a $concat | resources.Concat "all.css" | fingerprint }}
+all: {{ $all.RelPermalink }}
+-- layouts/index.html --
+{{ define "main" }}Main{{ end }}
+
+`
+	b := TestRunning(t, files)
+
+	b.AssertFileContent("public/index.html", "all: /all.ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad.css")
+	b.EditFileReplaceAll("assets/c.css", "c", "c edited").Build()
+	b.AssertFileContent("public/index.html", "all: /all.6dc8c9da5de8f1a85a6167148657c09f26be309eaa9527d757731f91e57e59e5.css")
+	b.AssertFileContent("public/all.6dc8c9da5de8f1a85a6167148657c09f26be309eaa9527d757731f91e57e59e5.css", "abc edited")
+}

identity/identity.go

@@ -312,7 +312,7 @@ func (im *identityManager) AddIdentity(ids ...Identity) {
 	im.mu.Lock()
 
 	for _, id := range ids {
-		if id == Anonymous {
+		if id == nil || id == Anonymous {
 			continue
 		}
 		if _, found := im.ids[id]; !found {
@@ -420,6 +420,9 @@ func walkIdentitiesShallow(v any, level int, cb func(level int, id Identity) boo
 		if id == Anonymous {
 			return false
 		}
+		if id == nil {
+			return false
+		}
 		return cb(level, id)
 	}
 

resources/resource_factories/bundler/bundler.go

@@ -20,6 +20,7 @@ import (
 	"path"
 
 	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/resource"
@@ -82,6 +83,11 @@ func (r *multiReadSeekCloser) Close() error {
 func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resource, error) {
 	targetPath = path.Clean(targetPath)
 	return c.rs.ResourceCache.GetOrCreate(targetPath, func() (resource.Resource, error) {
+		var idm identity.Manager = identity.NopManager
+		if c.rs.Cfg.Watching() {
+			idm = identity.NewManager("concat")
+		}
+
 		var resolvedm media.Type
 
 		// The given set of resources must be of the same Media Type.
@@ -91,6 +97,10 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou
 				return nil, fmt.Errorf("resources in Concat must be of the same Media Type, got %q and %q", r.MediaType().Type, resolvedm.Type)
 			}
 			resolvedm = r.MediaType()
+			identity.WalkIdentitiesShallow(r, func(depth int, id identity.Identity) bool {
+				idm.AddIdentity(id)
+				return false
+			})
 		}
 
 		concatr := func() (hugio.ReadSeekCloser, error) {
@@ -136,6 +146,7 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou
 				LazyPublish:        true,
 				OpenReadSeekCloser: concatr,
 				TargetPath:         targetPath,
+				DependencyManager:  idm,
 			})
 		if err != nil {
 			return nil, err

resources/resource_factories/create/create.go

@@ -64,7 +64,7 @@ func (c *Client) Copy(r resource.Resource, targetPath string) (resource.Resource
 }
 
 func (c *Client) newDependencyManager() identity.Manager {
-	if c.rs.Cfg.Running() {
+	if c.rs.Cfg.Watching() {
 		return identity.NewManager("resources")
 	}
 	return identity.NopManager