diff --git a/cache/memcache/memcache.go b/cache/memcache/memcache.go new file mode 100644 index 00000000000..7cfd363edac --- /dev/null +++ b/cache/memcache/memcache.go @@ -0,0 +1,460 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memcache provides the core memory cache used in Hugo. +package memcache + +import ( + "math" + "path" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/resources/resource" + + "github.com/gohugoio/hugo/helpers" + "github.com/pkg/errors" + + "github.com/BurntSushi/locker" + "github.com/karlseguin/ccache/v2" +) + +const ( + gigabyte = 1 << 30 +) + +const ( + ClearOnRebuild ClearWhen = iota + ClearOnChange + ClearNever +) + +type Getter interface { + GetOrCreate(path string, create func() Entry) (interface{}, error) + Clear() +} + +var errInvalidKey = errors.New("invalid cache key") + +// New creates a new cache. +func New(conf Config) *Cache { + if conf.TTL == 0 { + conf.TTL = time.Second * 33 + } + if conf.CheckInterval == 0 { + conf.CheckInterval = time.Second * 2 + } + if conf.MaxSize == 0 { + conf.MaxSize = 100000 + } + if conf.ItemsToPrune == 0 { + conf.ItemsToPrune = 200 + } + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + stats := &stats{ + memstatsStart: m, + maxSize: conf.MaxSize, + availableMemory: config.GetMemoryLimit(), + } + + conf.MaxSize = stats.adjustMaxSize(nil) + + c := &Cache{ + conf: conf, + cache: ccache.Layered(ccache.Configure().MaxSize(conf.MaxSize).ItemsToPrune(conf.ItemsToPrune)), + getters: make(map[string]*getter), + ttl: conf.TTL, + stats: stats, + nlocker: locker.NewLocker(), + } + + c.stop = c.start() + + return c +} + +// Cache configures a cache. +type Cache struct { + mu sync.Mutex + getters map[string]*getter + + conf Config + cache *ccache.LayeredCache + + ttl time.Duration + nlocker *locker.Locker + + stats *stats + stopOnce sync.Once + stop func() +} + +var extAliasKeywords = map[string]map[string]bool{ + "sass": map[string]bool{"scss": true}, + "js": map[string]bool{"ts": true}, +} + +func init() { + for k, v := range extAliasKeywords { + for vv, _ := range v { + if _, found := extAliasKeywords[vv]; !found { + m := make(map[string]bool) + m[k] = true + for vv2, _ := range v { + if vv2 != vv { + m[vv2] = true + } + } + extAliasKeywords[vv] = m + } + } + } + +} + +// ClearOn clears all the caches with that clearing strategy. +// This method is not thread safe. +func (c *Cache) ClearOn(when ClearWhen, changeset ...string) { + for _, g := range c.getters { + if g.clearWhen == ClearNever { + continue + } + + if g.clearWhen == when { + // Clear all. + g.Clear() + return + } + + changeset = helpers.UniqueStrings(changeset) + shouldDelete := func(key string, e Entry) bool { + if e.ClearWhen == when || resource.IsStaleAny(e, e.Value) { + return true + } + + if len(changeset) == 0 { + return false + } + + for _, filename := range changeset { + path := CleanKey(filename) + prefix, suffix := splitBasePathAndExt(path) + + // Will clear out all files that share a common root path, + // e.g "styles" in "styles/main.css". + if strings.HasPrefix(key, prefix) { + return true + } + + switch v := e.Value.(type) { + case resource.MediaTypeProvider: + // This will have some potential false positives, + // i.e. more work done, but this is supposed to + // evict entries for dependencies that Hugo isn't in fully + // control of, e.g.: + // A change in /assets/bootstrap/somefile.scss will trigger + // a rebuild of /assets/scss/main.scss. + checkSuffix := func(s string) bool { + return helpers.InStringArray(v.MediaType().Suffixes, s) + } + + if checkSuffix(suffix) { + return true + } + if aliases, found := extAliasKeywords[suffix]; found { + for alias, _ := range aliases { + if checkSuffix(alias) { + return true + } + } + } + } + } + + return false + } + + g.c.cache.DeleteFunc(g.partition, func(key string, item *ccache.Item) bool { + e := item.Value().(Entry) + if shouldDelete(key, e) { + resource.MarkStale(e.Value) + return true + } + return false + }) + + } +} + +// Clear clears the cache state. +// This method is not thread safe. +func (c *Cache) Clear() { + c.nlocker = locker.NewLocker() + c.cache.Clear() +} + +func (c *Cache) DeleteAll(primary string) bool { + return c.cache.DeleteAll(primary) +} + +func (c *Cache) GetDropped() int { + return c.cache.GetDropped() +} + +func (c *Cache) GetOrCreatePartition(partition string, clearWhen ClearWhen) Getter { + c.mu.Lock() + defer c.mu.Unlock() + + g, found := c.getters[partition] + if found { + if g.clearWhen != clearWhen { + panic("GetOrCreatePartition called with the same partition but different clearing strategy.") + } + return g + } + + g = &getter{ + partition: partition, + c: c, + clearWhen: clearWhen, + } + + c.getters[partition] = g + + return g +} + +func (c *Cache) Stop() { + c.stopOnce.Do(func() { + c.stop() + c.cache.Stop() + }) +} + +func (c *Cache) start() func() { + ticker := time.NewTicker(c.conf.CheckInterval) + quit := make(chan struct{}) + + checkAndAdjustMaxSize := func() { + var m runtime.MemStats + cacheDropped := c.GetDropped() + c.stats.decr(cacheDropped) + + runtime.ReadMemStats(&m) + c.stats.memstatsCurrent = m + c.stats.adjustMaxSize(c.cache.SetMaxSize) + + //fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) + + } + go func() { + for { + select { + case <-ticker.C: + checkAndAdjustMaxSize() + case <-quit: + ticker.Stop() + return + } + } + }() + + return func() { + close(quit) + } +} + +func (c *Cache) get(primary, secondary string) (interface{}, bool) { + v := c.cache.Get(primary, secondary) + if v == nil { + return nil, false + } + + return v.Value(), true +} + +// GetOrCreate tries to get the value with the given cache paths, if not found +// create will be called and cached. +// This method is thread safe. +func (c *Cache) getOrCreate(primary, secondary string, create func() Entry) (interface{}, error) { + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // The provided create function may be a relatively time consuming operation, + // and there will in the commmon case be concurrent requests for the same key'd + // resource, so make sure we pause these until the result is ready. + path := primary + secondary + c.nlocker.Lock(path) + defer c.nlocker.Unlock(path) + + // Try again. + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // Create it and store it in cache. + entry := create() + entry.size = 1 // For now. + if entry.Err != nil { + entry.ClearWhen = ClearOnRebuild + } + + c.cache.Set(primary, secondary, entry, c.ttl) + c.stats.incr(1) + + return entry.Value, entry.Err +} + +type ClearWhen int + +type Config struct { + CheckInterval time.Duration + MaxSize int64 + ItemsToPrune uint32 + TTL time.Duration + Running bool +} + +type Entry struct { + Value interface{} + size int64 + Err error + StaleFunc func() bool + ClearWhen +} + +func (e Entry) Size() int64 { + return e.size +} + +func (e Entry) IsStale() bool { + return e.StaleFunc != nil && e.StaleFunc() +} + +type getter struct { + c *Cache + partition string + + clearWhen ClearWhen +} + +func (g *getter) Clear() { + g.c.DeleteAll(g.partition) +} + +func (g *getter) GetOrCreate(path string, create func() Entry) (interface{}, error) { + return g.c.getOrCreate(g.partition, path, create) +} + +type stats struct { + memstatsStart runtime.MemStats + memstatsCurrent runtime.MemStats + maxSize int64 + availableMemory uint64 + numItems uint64 +} + +func (s *stats) resizeFactor() float64 { + if s.memstatsCurrent.Alloc == 0 { + return 1.0 + } + return math.Floor(float64(s.availableMemory/s.memstatsCurrent.Alloc)*10) / 10 +} + +func (s *stats) adjustMaxSize(setter func(size int64)) int64 { + newSize := int64(float64(s.maxSize) * s.resizeFactor()) + if newSize != s.maxSize && setter != nil { + setter(newSize) + } + return newSize +} + +func (s *stats) decr(i int) { + atomic.AddUint64(&s.numItems, ^uint64(i-1)) +} + +func (s *stats) incr(i int) { + atomic.AddUint64(&s.numItems, uint64(i)) +} + +// This assumes a valid key path. +func splitBasePathAndExt(path string) (string, string) { + dotIdx := strings.LastIndex(path, ".") + ext := path[dotIdx+1:] + slashIdx := strings.Index(path, "/") + + return path[:slashIdx], ext +} + +// CleanKey turns s into a format suitable for a cache key for this package. +// The key will be a Unix-styled path without any leading slash. +// If the input string does not contain any slash, a root will be prepended. +// If the input string does not contain any ".", a dummy file suffix will be appended. +// These are to make sure that they can effectively partake in the "cache cleaning" +// strategy used in server mode. +func CleanKey(s string) string { + s = path.Clean(helpers.ToSlashTrimLeading(s)) + if !strings.ContainsRune(s, '/') { + s = "_root/" + s + } + if !strings.ContainsRune(s, '.') { + s += "." + unknownExtension + } + + return s +} + +// InsertKeyPathElement inserts the given element after the first '/' in key. +func InsertKeyPathElements(key string, elements ...string) string { + slashIdx := strings.Index(key, "/") + return key[:slashIdx] + "/" + path.Join(elements...) + key[slashIdx:] +} + +const unknownExtension = "unkn" + +func keyValid(s string) bool { + if len(s) < 5 { + return false + } + if strings.ContainsRune(s, '\\') { + return false + } + if strings.HasPrefix(s, "/") { + return false + } + if !strings.ContainsRune(s, '/') { + return false + } + + dotIdx := strings.Index(s, ".") + if dotIdx == -1 || dotIdx == len(s)-1 { + return false + } + + return true +} diff --git a/cache/memcache/memcache_test.go b/cache/memcache/memcache_test.go new file mode 100644 index 00000000000..baad1e17222 --- /dev/null +++ b/cache/memcache/memcache_test.go @@ -0,0 +1,169 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memcache + +import ( + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + qt "github.com/frankban/quicktest" +) + +func TestCache(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{}) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + for i := 0; i < 5; i++ { + v1, err := cache.getOrCreate("a", "a1", create) + c.Assert(err, qt.IsNil) + c.Assert(v1, qt.Equals, 1) + v2, err := cache.getOrCreate("a", "a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v2, qt.Equals, 2) + } + + cache.Clear() + + v3, err := cache.getOrCreate("a", "a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v3, qt.Equals, 3) +} + +func TestCacheConcurrent(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + var wg sync.WaitGroup + + cache := New(Config{}) + + create := func(i int) func() Entry { + return func() Entry { + return Entry{Value: i} + } + } + + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + id := fmt.Sprintf("id%d", j) + v, err := cache.getOrCreate("a", id, create(j)) + c.Assert(err, qt.IsNil) + c.Assert(v, qt.Equals, j) + } + }() + } + wg.Wait() +} + +func TestCacheMemStats(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{ + ItemsToPrune: 10, + CheckInterval: 500 * time.Millisecond, + }) + + s := cache.stats + + c.Assert(s.memstatsStart.Alloc > 0, qt.Equals, true) + c.Assert(s.memstatsCurrent.Alloc, qt.Equals, uint64(0)) + c.Assert(s.availableMemory > 0, qt.Equals, true) + c.Assert(s.numItems, qt.Equals, uint64(0)) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + for i := 1; i <= 20; i++ { + _, err := cache.getOrCreate("a", fmt.Sprintf("b%d", i), create) + c.Assert(err, qt.IsNil) + } + + c.Assert(s.numItems, qt.Equals, uint64(20)) + cache.cache.SetMaxSize(10) + time.Sleep(time.Millisecond * 600) + c.Assert(int(s.numItems), qt.Equals, 10) + + c.Assert(s.memstatsCurrent.Alloc > 0, qt.Equals, true) +} + +func TestSplitBasePathAndExt(t *testing.T) { + t.Parallel() + c := qt.New(t) + + tests := []struct { + path string + a string + b string + }{ + {"a/b.json", "a", "json"}, + {"a/b/c/d.json", "a", "json"}, + } + for i, this := range tests { + msg := qt.Commentf("test %d", i) + a, b := splitBasePathAndExt(this.path) + + c.Assert(a, qt.Equals, this.a, msg) + c.Assert(b, qt.Equals, this.b, msg) + } + +} + +func TestCleanKey(t *testing.T) { + c := qt.New(t) + + c.Assert(CleanKey(filepath.FromSlash("a/b/c.js")), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a//b////c.js"), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a.js"), qt.Equals, "_root/a.js") + c.Assert(CleanKey("b/a"), qt.Equals, "b/a.unkn") + +} + +func TestKeyValid(t *testing.T) { + c := qt.New(t) + + c.Assert(keyValid("a/b.j"), qt.Equals, true) + c.Assert(keyValid("a/b."), qt.Equals, false) + c.Assert(keyValid("a/b"), qt.Equals, false) + c.Assert(keyValid("/a/b.txt"), qt.Equals, false) + c.Assert(keyValid("a\\b.js"), qt.Equals, false) + +} + +func TestInsertKeyPathElement(t *testing.T) { + c := qt.New(t) + + c.Assert(InsertKeyPathElements("a/b.j", "en"), qt.Equals, "a/en/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "en", "foo"), qt.Equals, "a/en/foo/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "", "foo"), qt.Equals, "a/foo/b.j") + +} diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go deleted file mode 100644 index d8c229a013b..00000000000 --- a/cache/namedmemcache/named_cache.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package namedmemcache provides a memory cache with a named lock. This is suitable -// for situations where creating the cached resource can be time consuming or otherwise -// resource hungry, or in situations where a "once only per key" is a requirement. -package namedmemcache - -import ( - "sync" - - "github.com/BurntSushi/locker" -) - -// Cache holds the cached values. -type Cache struct { - nlocker *locker.Locker - cache map[string]cacheEntry - mu sync.RWMutex -} - -type cacheEntry struct { - value interface{} - err error -} - -// New creates a new cache. -func New() *Cache { - return &Cache{ - nlocker: locker.NewLocker(), - cache: make(map[string]cacheEntry), - } -} - -// Clear clears the cache state. -func (c *Cache) Clear() { - c.mu.Lock() - defer c.mu.Unlock() - - c.cache = make(map[string]cacheEntry) - c.nlocker = locker.NewLocker() - -} - -// GetOrCreate tries to get the value with the given cache key, if not found -// create will be called and cached. -// This method is thread safe. It also guarantees that the create func for a given -// key is invoced only once for this cache. -func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) { - c.mu.RLock() - entry, found := c.cache[key] - c.mu.RUnlock() - - if found { - return entry.value, entry.err - } - - c.nlocker.Lock(key) - defer c.nlocker.Unlock(key) - - // Create it. - value, err := create() - - c.mu.Lock() - c.cache[key] = cacheEntry{value: value, err: err} - c.mu.Unlock() - - return value, err -} diff --git a/cache/namedmemcache/named_cache_test.go b/cache/namedmemcache/named_cache_test.go deleted file mode 100644 index 9feddb11f2a..00000000000 --- a/cache/namedmemcache/named_cache_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package namedmemcache - -import ( - "fmt" - "sync" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestNamedCache(t *testing.T) { - t.Parallel() - c := qt.New(t) - - cache := New() - - counter := 0 - create := func() (interface{}, error) { - counter++ - return counter, nil - } - - for i := 0; i < 5; i++ { - v1, err := cache.GetOrCreate("a1", create) - c.Assert(err, qt.IsNil) - c.Assert(v1, qt.Equals, 1) - v2, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v2, qt.Equals, 2) - } - - cache.Clear() - - v3, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v3, qt.Equals, 3) -} - -func TestNamedCacheConcurrent(t *testing.T) { - t.Parallel() - - c := qt.New(t) - - var wg sync.WaitGroup - - cache := New() - - create := func(i int) func() (interface{}, error) { - return func() (interface{}, error) { - return i, nil - } - } - - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - id := fmt.Sprintf("id%d", j) - v, err := cache.GetOrCreate(id, create(j)) - c.Assert(err, qt.IsNil) - c.Assert(v, qt.Equals, j) - } - }() - } - wg.Wait() -} diff --git a/commands/hugo.go b/commands/hugo.go index 5442c32d708..de311aaa646 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -433,8 +433,14 @@ func (c *commandeer) initMemTicker() func() { quit := make(chan struct{}) printMem := func() { var m runtime.MemStats + var cacheDropped int + h := c.hugo() + if h != nil && h.MemCache != nil { + cacheDropped = h.MemCache.GetDropped() + } + runtime.ReadMemStats(&m) - fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\n\n", formatByteCount(m.Alloc), formatByteCount(m.TotalAlloc), formatByteCount(m.Sys), m.NumGC) + fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) } @@ -1215,17 +1221,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { return name } - -func formatByteCount(b uint64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", - float64(b)/float64(div), "kMGTPE"[exp]) -} diff --git a/config/env.go b/config/env.go index f482cd24733..06233c4b641 100644 --- a/config/env.go +++ b/config/env.go @@ -17,6 +17,9 @@ import ( "os" "runtime" "strconv" + + "github.com/pbnjay/memory" + "strings" ) @@ -33,6 +36,38 @@ func GetNumWorkerMultiplier() int { return runtime.NumCPU() } +const ( + gigabyte = 1 << 30 +) + +// GetMemoryLimit returns the upper memory limit in bytes for Hugo's in-memory caches. +// Note that this does not represent "all of the memory" that Hugo will use, +// so it needs to be set to a lower number than the available system memory. +// It will read from the HUGO_MEMORYLIMIT (in Gigabytes) environment variable. +// If that is not set, it will set aside a quarter of the total system memory. +func GetMemoryLimit() uint64 { + if mem := os.Getenv("HUGO_MEMORYLIMIT"); mem != "" { + if v := stringToGibabyte(mem); v > 0 { + return v + } + + } + + m := memory.TotalMemory() + if m != 0 { + return uint64(m / 4) + } + + return 2 * gigabyte +} + +func stringToGibabyte(f string) uint64 { + if v, err := strconv.ParseFloat(f, 32); err == nil && v > 0 { + return uint64(v * gigabyte) + } + return 0 +} + // SetEnvVars sets vars on the form key=value in the oldVars slice. func SetEnvVars(oldVars *[]string, keyValues ...string) { for i := 0; i < len(keyValues); i += 2 { diff --git a/config/env_test.go b/config/env_test.go index 3c402b9ef8d..3b0faac3b7d 100644 --- a/config/env_test.go +++ b/config/env_test.go @@ -29,4 +29,6 @@ func TestSetEnvVars(t *testing.T) { key, val := SplitEnvVar("HUGO=rocks") c.Assert(key, qt.Equals, "HUGO") c.Assert(val, qt.Equals, "rocks") + + c.Assert(stringToGibabyte("1.0"), qt.Equals, uint64(1073741824)) } diff --git a/deps/deps.go b/deps/deps.go index 82a16ba5947..c7ef99b65f5 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -5,6 +5,8 @@ import ( "sync/atomic" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/cache/filecache" @@ -62,9 +64,12 @@ type Deps struct { // The configuration to use Cfg config.Provider `json:"-"` - // The file cache to use. + // The file caches to use. FileCaches filecache.Caches + // The memory cache to use. + MemCache *memcache.Cache + // The translation func to use Translate func(translationID string, args ...interface{}) string `json:"-"` @@ -158,6 +163,13 @@ type ResourceProvider interface { Clone(deps *Deps) error } +// Stop stops all running caches etc. +func (d *Deps) Stop() { + if d.MemCache != nil { + d.MemCache.Stop() + } +} + func (d *Deps) Tmpl() tpl.TemplateHandler { return d.tmpl } @@ -236,11 +248,12 @@ func New(cfg DepsCfg) (*Deps, error) { if err != nil { return nil, errors.WithMessage(err, "failed to create file caches from configuration") } + memCache := memcache.New(memcache.Config{Running: cfg.Running}) errorHandler := &globalErrHandler{} buildState := &BuildState{} - resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) + resourceSpec, err := resources.NewSpec(ps, fileCaches, memCache, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -277,6 +290,7 @@ func New(cfg DepsCfg) (*Deps, error) { Language: cfg.Language, Site: cfg.Site, FileCaches: fileCaches, + MemCache: memCache, BuildStartListeners: &Listeners{}, BuildState: buildState, Timeout: time.Duration(timeoutms) * time.Millisecond, @@ -311,7 +325,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er // The resource cache is global so reuse. // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) + d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.MemCache, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index 076d9161ccf..f001e36168b 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -371,9 +371,18 @@ Set `titleCaseStyle` to specify the title style used by the [title](/functions/t ## Configuration Environment Variables +### Control Parallel Processing + HUGO_NUMWORKERMULTIPLIER : Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used. +### Control Memory Usage + +HUGO_MEMORYLIMIT {{< new-in "0.75.0" >}} +: A value in Gigabytes (e.g. "1.5") to control the upper memory limit for Hugo's memory use. If not set, it will set aside a quarter of the total system memory. +Note that this does not represent a hard promise that Hugo will not use more memory than configured, but we will try our best. You may need to tune this setting depending on your site and/or system. + + ## Configuration Lookup Order Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior: diff --git a/go.mod b/go.mod index 161333daeca..88e1c771559 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93 github.com/gorilla/websocket v1.4.1 github.com/jdkato/prose v1.1.1 + github.com/karlseguin/ccache/v2 v2.0.7-0.20200814031513-0dbf3f125f13 github.com/kr/pretty v0.2.0 // indirect github.com/kyokomi/emoji v2.2.1+incompatible github.com/magefile/mage v1.9.0 @@ -39,6 +40,7 @@ require ( github.com/nicksnyder/go-i18n v1.10.0 github.com/niklasfasching/go-org v1.3.1 github.com/olekukonko/tablewriter v0.0.4 + github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 github.com/pelletier/go-toml v1.6.0 // indirect github.com/pkg/errors v0.9.1 github.com/rogpeppe/go-internal v1.5.1 diff --git a/go.sum b/go.sum index e47c224795f..85a555d97ff 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,22 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karlseguin/ccache v1.0.1 h1:0gpC6z1qtv0cKmsi5Su5tTB6bJ2vm9bfOLACpDEB/Ro= +github.com/karlseguin/ccache v1.0.2-0.20200216035407-d9aec58960c7 h1:JLz65tG+cbiJ/Yj46FF3rmVmH61g6TI8YWq8p0AX5Zc= +github.com/karlseguin/ccache v1.0.2-0.20200216035407-d9aec58960c7/go.mod h1:bm6z3svDxOYoWqVvk2JmnwOr6dtrTru4/MmlXksuQxk= +github.com/karlseguin/ccache v1.0.2-0.20200626122230-40275a30c888 h1:Pgq9C5Cc/VUYV3Jm9JhzWNPzj5ghtCZ+0CgvtVRx4w4= +github.com/karlseguin/ccache v1.0.2-0.20200626122230-40275a30c888/go.mod h1:bm6z3svDxOYoWqVvk2JmnwOr6dtrTru4/MmlXksuQxk= +github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= +github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= +github.com/karlseguin/ccache/v2 v2.0.6 h1:jFCLz4bF4EPfuCcvESAgYNClkEb31LV3WzyOwLlFz7w= +github.com/karlseguin/ccache/v2 v2.0.6/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200812003947-223703f7f0b5 h1:6eBqXgUzzakiY3AykWKzsqkf/qbaNJdOG5I31JxgiX0= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200812003947-223703f7f0b5/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200813023720-f63031fa4034 h1:rdkclwEmWUb5u/1hcFbrPEe+rDhILqJudpYm2HsCcs0= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200813023720-f63031fa4034/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200814031513-0dbf3f125f13 h1:IRtbBdf9nezZrJHWPJ1atlyY7hByAnLd/Vp0zrr3LV0= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200814031513-0dbf3f125f13/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -285,6 +301,8 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 h1:MfIUBZ1bz7TgvQLVa/yPJZOGeKEgs6eTKUjz3zB4B+U= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= @@ -388,6 +406,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/helpers/general.go b/helpers/general.go index 80e303087a5..377cba65578 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -61,6 +61,24 @@ func FindAvailablePort() (*net.TCPAddr, error) { return nil, err } +// FormatByteCount pretty formats b. +func FormatByteCount(bc uint64) string { + const ( + Gigabyte = 1 << 30 + Megabyte = 1 << 20 + Kilobyte = 1 << 10 + ) + switch { + case bc > Gigabyte || -bc > Gigabyte: + return fmt.Sprintf("%.2f GB", float64(bc)/Gigabyte) + case bc > Megabyte || -bc > Megabyte: + return fmt.Sprintf("%.2f MB", float64(bc)/Megabyte) + case bc > Kilobyte || -bc > Kilobyte: + return fmt.Sprintf("%.2f KB", float64(bc)/Kilobyte) + } + return fmt.Sprintf("%d B", bc) +} + // InStringArray checks if a string is an element of a slice of strings // and returns a boolean value. func InStringArray(arr []string, el string) bool { diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 79d89a88b56..9bd44185712 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -40,6 +40,7 @@ const ( metaKeyBaseDir = "baseDir" // Abs base directory of source file. metaKeyMountRoot = "mountRoot" metaKeyModule = "module" + metaKeyComponent = "component" metaKeyOriginalFilename = "originalFilename" metaKeyName = "name" metaKeyPath = "path" @@ -132,6 +133,10 @@ func (f FileMeta) MountRoot() string { return f.stringV(metaKeyMountRoot) } +func (f FileMeta) Component() string { + return f.stringV(metaKeyComponent) +} + func (f FileMeta) Module() string { return f.stringV(metaKeyModule) } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index ea3ef003ecc..a031f1ec04d 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -41,8 +41,8 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { for _, rm := range rms { (&rm).clean() - fromBase := files.ResolveComponentFolder(rm.From) - if fromBase == "" { + component := files.ResolveComponentFolder(rm.From) + if component == "" { panic("unrecognised component folder in" + rm.From) } @@ -58,7 +58,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { return nil, err } // Extract "blog" from "content/blog" - rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) + rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, component), filepathSeparator) if rm.Meta == nil { rm.Meta = make(FileMeta) } @@ -66,6 +66,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rm.Meta[metaKeyBaseDir] = rm.ToBasedir rm.Meta[metaKeyMountRoot] = rm.path rm.Meta[metaKeyModule] = rm.Module + rm.Meta[metaKeyComponent] = component meta := copyFileMeta(rm.Meta) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index b32f808c971..a7af2a290c0 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -251,6 +251,13 @@ func (m *pageMap) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resour } target := strings.TrimPrefix(meta.Path(), owner.File().Dir()) + tbase := meta.TranslationBaseNameWithExt() + if tbase != "" { + dir, base := filepath.Split(target) + if base != tbase { + target = filepath.Join(dir, tbase) + } + } return owner.s.ResourceSpec.New( resources.ResourceSourceDescriptor{ diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 57a95a03713..8a735f49f51 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -20,6 +20,7 @@ import ( "os" "path" "path/filepath" + "sort" "strings" "sync" @@ -238,6 +239,19 @@ func (s SourceFilesystems) IsAsset(filename string) bool { return s.Assets.Contains(filename) } +// CollectResourcePaths collects paths relative to their component root from +// the data, content and assets filesystems. +func (s SourceFilesystems) CollectResourcePaths(filename string) []string { + var paths []string + + for _, fs := range []*SourceFilesystem{s.Assets, s.Content, s.Data} { + paths = append(paths, fs.collectRelativePaths(filename)...) + } + + return uniqueStringsSorted(paths) + +} + // IsI18n returns true if the given filename is a member of the i18n filesystem. func (s SourceFilesystems) IsI18n(filename string) bool { return s.I18n.Contains(filename) @@ -257,21 +271,40 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { // MakePathRelative creates a relative path from the given filename. // It will return an empty string if the filename is not a member of this filesystem. +// TODO(bep) there may be multiple, see https://github.com/gohugoio/hugo/issues/7563 +// See collectRelativePaths func (d *SourceFilesystem) MakePathRelative(filename string) string { + paths := d.collectRelativePaths(filename) + if paths == nil { + return "" + } + return paths[0] + +} +func (d *SourceFilesystem) collectRelativePaths(filename string) []string { + var paths []string for _, dir := range d.Dirs { meta := dir.(hugofs.FileMetaInfo).Meta() currentPath := meta.Filename() - - if strings.HasPrefix(filename, currentPath) { - rel := strings.TrimPrefix(filename, currentPath) + if rel := relFilename(currentPath, filename); rel != "" { if mp := meta.Path(); mp != "" { rel = filepath.Join(mp, rel) } - return strings.TrimPrefix(rel, filePathSeparator) + paths = append(paths, rel) } } - return "" + return paths +} + +func relFilename(dirname, filename string) string { + if !strings.HasSuffix(dirname, filePathSeparator) { + dirname += filePathSeparator + } + if !strings.HasPrefix(filename, dirname) { + return "" + } + return strings.TrimPrefix(filename, dirname) } func (d *SourceFilesystem) RealFilename(rel string) string { @@ -743,3 +776,21 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollect return b.createOverlayFs(collector, mounts[1:]) } + +func uniqueStringsSorted(s []string) []string { + if len(s) == 0 { + return nil + } + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 67ee10e0978..b1d94c4eed4 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -51,6 +51,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Make sure we don't trigger rebuilds in parallel. h.runningMu.Lock() defer h.runningMu.Unlock() + } else { + defer func() { + h.Stop() + }() } ctx, task := trace.NewTask(context.Background(), "Build") diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 8d0872bd5e0..88ac32fa10b 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -2,6 +2,7 @@ package hugolib import ( "fmt" + "os" "strings" "testing" @@ -11,6 +12,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource_transformers/tocss/scss" "github.com/fortytw2/leaktest" "github.com/fsnotify/fsnotify" @@ -417,15 +419,18 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { func TestMultiSitesRebuild(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 - // This leaktest seems to be a little bit shaky on Travis. - if !isCI() { - defer leaktest.CheckTimeout(t, 10*time.Second)() - } c := qt.New(t) b := newMultiSiteTestDefaultBuilder(t).Running().CreateSites().Build(BuildCfg{}) + defer b.H.Stop() + + // This leaktest seems to be a little bit shaky on Travis. + if !isCI() { + defer leaktest.CheckTimeout(t, 10*time.Second)() + } + sites := b.H.Sites fs := b.Fs @@ -1478,3 +1483,71 @@ func TestRebuildOnAssetChange(t *testing.T) { b.Build(BuildCfg{}) b.AssertFileContent("public/index.html", `changed data`) } + +func TestCacheOnRebuild(t *testing.T) { + c := qt.New(t) + + createSitesBuilder := func(t testing.TB, prepare func(b *sitesBuilder)) *sitesBuilder { + b := newTestSitesBuilder(t).Running() + prepare(b) + b.Build(BuildCfg{}) + return b + + } + + c.Run("transform.Unmarshal with temporary Resource", func(c *qt.C) { + templ := ` + {{ $m := "a: 1" | resources.FromString "a.yaml" | transform.Unmarshal }} + NM: {{ $m }} + + ` + b := createSitesBuilder(c, func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", templ) + }) + + b.AssertFileContent("public/index.html", `NM: map[a:1]`) + + templ = strings.ReplaceAll(templ, "a: 1", "a: 2") + templ = strings.ReplaceAll(templ, "NM:", "NM2:") + b.EditFiles("layouts/index.html", templ) + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `NM2: map[a:2]`) + + }) + + c.Run("SCSS with multiple roots", func(c *qt.C) { + if !scss.Supports() { + c.Skip("Skip SCSS") + } + + b, workDir, clean := newTestSitesBuilderWithOSFs(c, "scss-multiroot") + defer clean() + b.Running() + + assetsDir := filepath.Join(workDir, "assets") + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root1"), 0777), qt.IsNil) + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root2"), 0777), qt.IsNil) + + b.WithTemplatesAdded("index.html", ` +{{ $css := resources.Get "root2/main.scss" | toCSS }} +CSS: {{ $css.Content | safeHTML }} +`) + + b.WithSourceFile(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: blue;`) + b.WithSourceFile(filepath.Join(assetsDir, "root2/main.scss"), `@import "../root1/_vars"; + +body { color: $base-color; } +`) + + b.Build(BuildCfg{}) + b.AssertFileContent("public/index.html", `color: blue;`) + + b.EditFiles(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: red;`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `color: red;`) + }) + +} diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 4fe4960422c..80885b5d67c 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -89,7 +89,7 @@ languageName = "Nynorsk" s2h := s2.getPage(page.KindHome) c.Assert(s2h.Permalink(), qt.Equals, "https://example.fr/") - b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /docs/text/pipes.txt") + b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /text/pipes.txt") b.AssertFileContent("public/fr/text/pipes.txt", "Hugo Pipes") b.AssertFileContent("public/en/index.html", "Default Home Page", "String Resource: /docs/text/pipes.txt") b.AssertFileContent("public/en/text/pipes.txt", "Hugo Pipes") diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index fa420a025aa..5af55e738f0 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -347,7 +347,7 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { b.AssertFileContent("public/en/bc/data1.json", "data1") b.AssertFileContent("public/en/bc/data2.json", "data2") b.AssertFileContent("public/en/bc/logo-bc.png", "logo") - b.AssertFileContent("public/nn/bc/data1.nn.json", "data1.nn") + b.AssertFileContent("public/nn/bc/data1.json", "data1.nn") b.AssertFileContent("public/nn/bc/data2.json", "data2") b.AssertFileContent("public/nn/bc/logo-bc.png", "logo") @@ -1320,7 +1320,7 @@ bundle min min key: {{ $jsonMinMin.Key }} b.AssertFileContent(index, fmt.Sprintf("bundle min min min: /bundle%d/data.min.min.min.json", i), - fmt.Sprintf("bundle min min key: /bundle%d/data.min.min.json", i), + fmt.Sprintf("bundle min min key: bundle%d/content/en/data_c9a5f8c93697691d6f8a298931d285f7.json", i), ) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.min.min.json", i)), qt.Equals, true) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.json", i)), qt.Equals, false) diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 020e243c505..f5183f2260b 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -178,10 +178,6 @@ func (c *pagesCollector) Collect() (collectErr error) { } for dir := range dirs { - for _, pm := range c.contentMap.pmaps { - pm.s.ResourceSpec.DeleteBySubstring(dir.dirname) - } - switch dir.tp { case bundleLeaf: collectErr = c.collectDir(dir.dirname, true, nil) diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 97d4f17bab7..066c788fd16 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -53,6 +53,8 @@ type Paths struct { PublishDir string + IsMultiHost bool + // When in multihost mode, this returns a list of base paths below PublishDir // for each language. MultihostTargetBasePaths []string @@ -142,7 +144,8 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { } var multihostTargetBasePaths []string - if languages.IsMultihost() { + isMultiHost := languages.IsMultihost() + if isMultiHost { for _, l := range languages { multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang) } @@ -171,6 +174,7 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { Language: language, Languages: languages, LanguagesDefaultFirst: languagesDefaultFirst, + IsMultiHost: isMultiHost, MultihostTargetBasePaths: multihostTargetBasePaths, PaginatePath: cfg.GetString("paginatePath"), diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index c687ca3421b..e208d4d2ed2 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -350,7 +350,7 @@ Edited content. `) b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil) - b.H.ResourceSpec.ClearCaches() + b.H.MemCache.Clear() } } diff --git a/hugolib/resource_change_test.go b/hugolib/resource_change_test.go new file mode 100644 index 00000000000..941df8e7ea0 --- /dev/null +++ b/hugolib/resource_change_test.go @@ -0,0 +1,322 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "strings" + "testing" +) + +func TestResourceEditMetadata(t *testing.T) { + b := newTestSitesBuilder(t).Running() + + content := `+++ +title = "My Bundle With TOML Meta" + +[[resources]] +src = "**.toml" +title = "My TOML :counter" ++++ + +Content. +` + + b.WithContent( + "bundle/index.md", content, + "bundle/my1.toml", `a = 1`, + "bundle/my2.toml", `a = 2`) + + b.WithTemplatesAdded("index.html", ` +{{ $bundle := site.GetPage "bundle" }} +{{ $toml := $bundle.Resources.GetMatch "*.toml" }} +TOML: {{ $toml.Title }} + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My TOML 1") + + b.EditFiles("content/bundle/index.md", strings.ReplaceAll(content, "My TOML", "My Changed TOML 1")) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My Changed TOML") + +} + +func TestResourceCacheSimpleTest(t *testing.T) { + conf := ` +baseURL = "https://example.org" + +defaultContentLanguage = "en" + +[module] +[[module.mounts]] +source = "content/cen" +target = "content" +lang="en" +[[module.mounts]] +source = "content/cno" +target = "content" +lang="no" +[[module.mounts]] +source = "assets" +target = "assets" +[[module.mounts]] +source = "assets_common" +target = "assets/aen" +[[module.mounts]] +source = "assets_common" +target = "assets/ano" + +[languages] +[languages.en] +weight = 1 + +[languages.no] +weight = 2 + +` + b := newTestSitesBuilder(t).WithConfigFile("toml", conf).Running() + + b.WithSourceFile( + "content/cen/bundle/index.md", "---\ntitle: En Bundle\n---", + "content/cen/bundle/data1.json", `{ "data1": "en" }`, + "content/cen/bundle/data2.json", `{ "data2": "en" }`, + "content/cno/bundle/index.md", "---\ntitle: No Bundle\n---", + "content/cno/bundle/data1.json", `{ "data1": "no" }`, + "content/cno/bundle/data3.json", `{ "data3": "no" }`, + ) + + b.WithSourceFile("assets_common/data/common.json", `{ + "Hugo": "Rocks!", + }`) + + b.WithSourceFile("assets/data/mydata.json", `{ + "a": 32, + }`) + + b.WithTemplatesAdded("index.html", ` +{{ $data := resources.Get "data/mydata.json" }} +{{ template "print-resource" ( dict "title" "data" "r" $data ) }} +{{ $dataMinified := $data | minify }} +{{ template "print-resource" ( dict "title" "data-minified" "r" $dataMinified ) }} +{{ $dataUnmarshaled := $dataMinified | transform.Unmarshal }} +Data Unmarshaled: {{ $dataUnmarshaled }} +{{ $bundle := site.GetPage "bundle" }} +{{ range (seq 3) }} +{{ $i := . }} +{{ with $bundle.Resources.GetMatch (printf "data%d.json" . ) }} +{{ $minified := . | minify }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d" $i) "r" . ) }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d min" $i) "r" $minified ) }} +{{ end }} +{{ end }} +{{ $common1 := resources.Get "aen/data/common.json" }} +{{ $common2 := resources.Get "ano/data/common.json" }} +{{ template "print-resource" ( dict "title" "common1" "r" $common1 ) }} +{{ template "print-resource" ( dict "title" "common2" "r" $common2 ) }} +{{ define "print-resource" }}{{ .title }}|{{ .r.RelPermalink }}|{{ .r.Key }}|{{ .r.Content | safeHTML }}|{{ end }} + + + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +Data Unmarshaled: map[a:32] +bundle data 1|/bundle/data1.json|bundle/content/en/data1.json|{ "data1": "en" }| +bundle data 1 min|/bundle/data1.min.json|bundle/content/en/data1_d3f53f09220d597dac26fe7840c31fc9.json|{"data1":"en"}| +bundle data 3|/bundle/data3.json|bundle/content/en/data3.json|{ "data3": "no" }| +bundle data 3 min|/bundle/data3.min.json|bundle/content/en/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json| +common2|/ano/data/common.json|ano/data/common.json| +`) + + b.AssertFileContent("public/no/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +bundle data 1|/no/bundle/data1.json|bundle/content/no/data1.json|{ "data1": "no" }| +bundle data 2|/no/bundle/data2.json|bundle/content/no/data2.json|{ "data2": "en" }| + bundle data 3|/no/bundle/data3.json|bundle/content/no/data3.json|{ "data3": "no" }| +bundle data 3 min|/no/bundle/data3.min.json|bundle/content/no/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json +`) + + b.EditFiles("assets/data/mydata.json", `{ "a": 42 }`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data|/data/mydata.json|data/mydata.json|{ "a": 42 }| +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":42}| +Data Unmarshaled: map[a:42] +`) + +} + +func TestResourceCacheMultihost(t *testing.T) { + toLang := func(format, lang string) string { + return fmt.Sprintf(format, lang) + } + + addContent := func(b *sitesBuilder, contentPath func(path, lang string) string) { + b.WithNoContentAdded() + for _, lang := range []string{"en", "fr"} { + b.WithSourceFile( + contentPath("b1/index.md", lang), toLang("---\ntitle: Bundle 1 %s\n---", lang), + contentPath("b1/styles/style11.css", lang), toLang(".%s1: { color: blue };", lang), + contentPath("b1/styles/style12.css", lang), toLang(".%s2: { color: red };", lang), + ) + b.WithSourceFile( + contentPath("b2/index.md", lang), toLang("---\ntitle: Bundle 2 %s\n---", lang), + contentPath("b2/styles/style21.css", lang), toLang(".%s21: { color: green };", lang), + contentPath("b2/styles/style22.css", lang), toLang(".%s22: { color: orange };", lang), + ) + } + } + + addTemplates := func(b *sitesBuilder) { + b.WithTemplates("_default/single.html", ` +{{ template "print-page" (dict "page" . "title" "Self") }} +{{ $other := site.Sites.First.GetPage "b1" }} +{{ template "print-page" (dict "page" $other "title" "Other") }} + + +{{ define "print-page" }} +{{ $p := .page }} +{{ $title := .title }} +{{ $styles := $p.Resources.Match "**/style1*.css" }} +{{ if $styles }} +{{ $firststyle := index $styles 0 }} +{{ $mystyles := $styles | resources.Concat "mystyles.css" }} +{{ $title }} Mystyles First CSS: {{ $firststyle.RelPermalink }}|Key: {{ $firststyle.Key }}|{{ $firststyle.Content }}| +{{ $title }} Mystyles CSS: {{ $mystyles.RelPermalink }}|Key: {{ $mystyles.Key }}|{{ $mystyles.Content }}| +{{ end }} +{{ $title }} Bundle: {{ $p.Permalink }} +{{ $style := $p.Resources.GetMatch "**.css" }} +{{ $title }} CSS: {{ $style.RelPermalink }}|Key: {{ $style.Key }}|{{ $style.Content }}| +{{ $minified := $style | minify }} +{{ $title }} Minified CSS: {{ $minified.RelPermalink }}|Key: {{ $minified.Key }}|{{ $minified.Content }} +{{ end }} + +`) + } + + assertContent := func(b *sitesBuilder) { + + otherAssert := `Other Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Other Bundle: https://example.com/b1/ +Other CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue }; +Other Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue}` + + b.AssertFileContent("public/fr/b1/index.html", ` +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red }; +Self Bundle: https://example.fr/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/fr/styles/style11.css|.fr1: { color: blue }; +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/fr/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.fr1:{color:blue} +`, + otherAssert) + b.AssertFileContent("public/en/b1/index.html", ` +Self Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red };| +Self Bundle: https://example.com/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue} +`, otherAssert) + + b.AssertFileContent("public/fr/b2/index.html", ` +Self Bundle: https://example.fr/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/fr/styles/style21.css|.fr21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/fr/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.fr21:{color:green} +`, otherAssert) + + b.AssertFileContent("public/en/b2/index.html", ` +Self Bundle: https://example.com/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/en/styles/style21.css|.en21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/en/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.en21:{color:green} +`, otherAssert) + + } + + t.Run("Default content", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false +contentDir = "content" + +[Languages] +[Languages.en] +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + fmt.Println(b.workingDir) + addContent(b, func(path, lang string) string { + path = strings.Replace(path, ".", "."+lang+".", 1) + path = "content/" + path + return path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + + t.Run("Content dir per language", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false + +[Languages] +[Languages.en] +contentDir = "content_en" +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +contentDir = "content_fr" +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + addContent(b, func(path, lang string) string { + return "content_" + lang + "/" + path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + +} diff --git a/hugolib/site.go b/hugolib/site.go index ac65931d0c8..fe6194c93c6 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -28,12 +28,12 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/constants" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/converter/hooks" @@ -1027,11 +1027,12 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro logger = helpers.NewDistinctFeedbackLogger() ) - var cachePartitions []string + // Paths relative to their component folder in data, assets or content. + var resourcePaths []string for _, ev := range events { - if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { - cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) + if paths := s.BaseFs.CollectResourcePaths(ev.Name); paths != nil { + resourcePaths = append(resourcePaths, paths...) } id, found := s.eventToIdentity(ev) @@ -1075,10 +1076,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro return err } - // These in memory resource caches will be rebuilt on demand. - for _, s := range s.h.Sites { - s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) - } + h.MemCache.ClearOn(memcache.ClearOnRebuild, resourcePaths...) if tmplChanged || i18nChanged { sites := s.h.Sites diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 2af4691d173..fb17738ad7d 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -92,7 +92,8 @@ type sitesBuilder struct { // Consider this in relation to using the BaseFs.PublishFs to all publishing. workingDir string - addNothing bool + addNothing bool + addNoContent bool // Base data/content contentFilePairs []filenameContent templateFilePairs []filenameContent @@ -126,6 +127,21 @@ func newTestSitesBuilder(t testing.TB) *sitesBuilder { dumper: litterOptions, rnd: rand.New(rand.NewSource(time.Now().Unix()))} } +func newTestSitesBuilderWithOSFs(t testing.TB, testname string) (*sitesBuilder, string, func()) { + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, testname) + if err != nil { + t.Fatal(err) + } + v := viper.New() + v.Set("workingDir", workDir) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) + + return b, workDir, clean +} + func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder { c := qt.New(t) @@ -154,6 +170,11 @@ func (s *sitesBuilder) WithNothingAdded() *sitesBuilder { return s } +func (s *sitesBuilder) WithNoContentAdded() *sitesBuilder { + s.addNoContent = true + return s +} + func (s *sitesBuilder) WithLogger(logger *loggers.Logger) *sitesBuilder { s.logger = logger return s @@ -502,6 +523,7 @@ func (s *sitesBuilder) LoadConfig() error { } func (s *sitesBuilder) CreateSitesE() error { + if !s.addNothing { if _, ok := s.Fs.Source.(*afero.OsFs); ok { for _, dir := range []string{ @@ -529,7 +551,6 @@ func (s *sitesBuilder) CreateSitesE() error { s.writeFilePairs("data", s.dataFilePairs) s.writeFilePairs("content", s.contentFilePairs) s.writeFilePairs("layouts", s.templateFilePairs) - } if err := s.LoadConfig(); err != nil { @@ -677,7 +698,7 @@ hello: } ) - if len(s.contentFilePairs) == 0 { + if len(s.contentFilePairs) == 0 && !s.addNoContent { s.writeFilePairs("content", s.createFilenameContent(defaultContent)) } diff --git a/modules/client.go b/modules/client.go index 8ee9f588aaf..a13e7688105 100644 --- a/modules/client.go +++ b/modules/client.go @@ -342,7 +342,7 @@ var verifyErrorDirRe = regexp.MustCompile(`dir has been modified \((.*?)\)`) // which are stored in a local downloaded source cache, have not been // modified since being downloaded. func (c *Client) Verify(clean bool) error { - // TODO1 add path to mod clean + // TODO(bep) add path to mod clean err := c.runVerify() if err != nil { diff --git a/resources/image.go b/resources/image.go index d1d9f650d67..5b3905de67d 100644 --- a/resources/image.go +++ b/resources/image.go @@ -367,7 +367,7 @@ func (i *imageResource) getImageMetaCacheTargetPath() string { df.dir = filepath.Dir(fi.Meta().Path()) } p1, _ := helpers.FileAndExt(df.file) - h, _ := i.hash() + h := i.hash() idStr := helpers.HashString(h, i.size(), imageMetaVersionNumber, cfg) return path.Join(df.dir, fmt.Sprintf("%s_%s.json", p1, idStr)) } @@ -378,7 +378,7 @@ func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile p2 = conf.TargetFormat.DefaultExtension() } - h, _ := i.hash() + h := i.hash() idStr := fmt.Sprintf("_hu%s_%d", h, i.size()) // Do not change for no good reason. diff --git a/resources/image_cache.go b/resources/image_cache.go index 1888b457f59..b1db71cbea6 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -17,8 +17,8 @@ import ( "image" "io" "path/filepath" - "strings" - "sync" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/images" @@ -30,36 +30,7 @@ type imageCache struct { pathSpec *helpers.PathSpec fileCache *filecache.Cache - - mu sync.RWMutex - store map[string]*resourceAdapter -} - -func (c *imageCache) deleteIfContains(s string) { - c.mu.Lock() - defer c.mu.Unlock() - s = c.normalizeKeyBase(s) - for k := range c.store { - if strings.Contains(k, s) { - delete(c.store, k) - } - } -} - -// The cache key is a lowecase path with Unix style slashes and it always starts with -// a leading slash. -func (c *imageCache) normalizeKey(key string) string { - return "/" + c.normalizeKeyBase(key) -} - -func (c *imageCache) normalizeKeyBase(key string) string { - return strings.Trim(strings.ToLower(filepath.ToSlash(key)), "/") -} - -func (c *imageCache) clear() { - c.mu.Lock() - defer c.mu.Unlock() - c.store = make(map[string]*resourceAdapter) + mCache memcache.Getter } func (c *imageCache) getOrCreate( @@ -67,101 +38,92 @@ func (c *imageCache) getOrCreate( createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) { relTarget := parent.relTargetPathFromConfig(conf) memKey := parent.relTargetPathForRel(relTarget.path(), false, false, false) - memKey = c.normalizeKey(memKey) + memKey = memcache.CleanKey(memKey) - // For the file cache we want to generate and store it once if possible. - fileKeyPath := relTarget - if fi := parent.root.getFileInfo(); fi != nil { - fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) - } - fileKey := fileKeyPath.path() + v, err := c.mCache.GetOrCreate(memKey, func() memcache.Entry { + // For the file cache we want to generate and store it once if possible. + fileKeyPath := relTarget + if fi := parent.root.getFileInfo(); fi != nil { + fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) + } + fileKey := fileKeyPath.path() - // First check the in-memory store, then the disk. - c.mu.RLock() - cachedImage, found := c.store[memKey] - c.mu.RUnlock() + var img *imageResource - if found { - return cachedImage, nil - } + // These funcs are protected by a named lock. + // read clones the parent to its new name and copies + // the content to the destinations. + read := func(info filecache.ItemInfo, r io.ReadSeeker) error { + img = parent.clone(nil) + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - var img *imageResource + if err := img.InitConfig(r); err != nil { + return err + } - // These funcs are protected by a named lock. - // read clones the parent to its new name and copies - // the content to the destinations. - read := func(info filecache.ItemInfo, r io.ReadSeeker) error { - img = parent.clone(nil) - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) + r.Seek(0, 0) - if err := img.InitConfig(r); err != nil { - return err - } + w, err := img.openDestinationsForWriting() + if err != nil { + return err + } - r.Seek(0, 0) + if w == nil { + // Nothing to write. + return nil + } + + defer w.Close() + _, err = io.Copy(w, r) - w, err := img.openDestinationsForWriting() - if err != nil { return err } - if w == nil { - // Nothing to write. - return nil - } + // create creates the image and encodes it to the cache (w). + create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + defer w.Close() - defer w.Close() - _, err = io.Copy(w, r) + var conv image.Image + img, conv, err = createImage() + if err != nil { + return + } + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - return err - } + return img.EncodeTo(conf, conv, w) + } + + // Now look in the file cache. - // create creates the image and encodes it to the cache (w). - create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { - defer w.Close() + // The definition of this counter is not that we have processed that amount + // (e.g. resized etc.), it can be fetched from file cache, + // but the count of processed image variations for this site. + c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - var conv image.Image - img, conv, err = createImage() + _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { - return + return memcache.Entry{Err: err} } - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) - return img.EncodeTo(conf, conv, w) - } + // The file is now stored in this cache. + img.setSourceFs(c.fileCache.Fs) - // Now look in the file cache. + imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - // The definition of this counter is not that we have processed that amount - // (e.g. resized etc.), it can be fetched from file cache, - // but the count of processed image variations for this site. - c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) + return memcache.Entry{Value: imgAdapter} + }) - _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { return nil, err } + return v.(*resourceAdapter), nil - // The file is now stored in this cache. - img.setSourceFs(c.fileCache.Fs) - - c.mu.Lock() - if cachedImage, found = c.store[memKey]; found { - c.mu.Unlock() - return cachedImage, nil - } - - imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - c.store[memKey] = imgAdapter - c.mu.Unlock() - - return imgAdapter, nil } -func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { - return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)} +func newImageCache(fileCache *filecache.Cache, memCache *memcache.Cache, ps *helpers.PathSpec) *imageCache { + return &imageCache{fileCache: fileCache, mCache: memCache.GetOrCreatePartition("images", memcache.ClearOnChange), pathSpec: ps} } diff --git a/resources/image_test.go b/resources/image_test.go index f98d9f4bb24..2826ea98405 100644 --- a/resources/image_test.go +++ b/resources/image_test.go @@ -352,7 +352,7 @@ func TestImageResizeInSubPath(t *testing.T) { c.Assert(spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) // Cleare mem cache to simulate reading from the file cache. - spec.imageCache.clear() + spec.imageCache.mCache.Clear() resizedAgain, err := image.Resize("101x101") c.Assert(err, qt.IsNil) diff --git a/resources/resource.go b/resources/resource.go index acdf2d744ec..5209c0a9796 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -22,6 +22,8 @@ import ( "path/filepath" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/resources/internal" "github.com/gohugoio/hugo/common/herrors" @@ -172,8 +174,6 @@ func (commonResource) Slice(in interface{}) (interface{}, error) { return nil, fmt.Errorf("type %T is not a Resource", v) } groups[i] = g - { - } } return groups, nil default: @@ -197,8 +197,24 @@ type fileInfo interface { setSourceFilename(string) setSourceFs(afero.Fs) getFileInfo() hugofs.FileMetaInfo - hash() (string, error) size() int + hashProvider +} + +type hashProvider interface { + hash() string +} + +type resourceOrigin struct { + stale bool +} + +func (s *resourceOrigin) MarkStale() { + s.stale = true +} + +func (s *resourceOrigin) IsStale() bool { + return s.stale } // genericResource represents a generic linkable resource. @@ -235,7 +251,25 @@ func (l *genericResource) Data() interface{} { } func (l *genericResource) Key() string { - return l.RelPermalink() + // TODO1 consider repating the section in the path segment. + + if l.fi != nil { + // Create a key that at least shares the base folder with the source, + // to facilitate effective cache busting on changes. + meta := l.fi.Meta() + p := meta.Path() + if p != "" { + d, _ := filepath.Split(p) + p = path.Join(d, l.relTargetDirFile.file) + key := memcache.CleanKey(p) + key = memcache.InsertKeyPathElements(key, meta.Component(), meta.Lang()) + + return key + } + } + key := memcache.CleanKey(l.RelPermalink()) + + return key } func (l *genericResource) MediaType() media.Type { @@ -417,6 +451,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour } fpath, fname := path.Split(u.targetPath) + r.resourcePathDescriptor.relTargetDirFile = dirFile{dir: fpath, file: fname} r.mergeData(u.data) @@ -623,26 +658,25 @@ func (fi *resourceFileInfo) setSourceFs(fs afero.Fs) { fi.sourceFs = fs } -func (fi *resourceFileInfo) hash() (string, error) { +func (fi *resourceFileInfo) hash() string { var err error fi.h.init.Do(func() { var hash string var f hugio.ReadSeekCloser f, err = fi.ReadSeekCloser() if err != nil { - err = errors.Wrap(err, "failed to open source file") - return + panic(fmt.Sprintf("failed to open source file: %s", err)) } defer f.Close() hash, err = helpers.MD5FromFileFast(f) if err != nil { - return + panic(err) } fi.h.value = hash }) - return fi.h.value, err + return fi.h.value } func (fi *resourceFileInfo) size() int { diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 13ffc5ae3ea..9f4dfdedf29 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -179,6 +179,36 @@ type UnmarshableResource interface { Identifier } +// Staler controls stale state of a Resource. A stale resource should be discarded. +type Staler interface { + MarkStale() + StaleInfo +} + +// StaleInfo tells if a resource is marked as stale. +type StaleInfo interface { + IsStale() bool +} + +// IsStaleAny reports whether any of the os is marked as stale. +func IsStaleAny(os ...interface{}) bool { + for _, o := range os { + if s, ok := o.(StaleInfo); ok && s.IsStale() { + return true + } + } + return false +} + +// MarkStale will mark any of the oses as stale, if possible. +func MarkStale(os ...interface{}) { + for _, o := range os { + if s, ok := o.(Staler); ok { + s.MarkStale() + } + } +} + type resourceTypesHolder struct { mediaType media.Type resourceType string diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 47822a7f506..e36d0435471 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -16,25 +16,13 @@ package resources import ( "encoding/json" "io" - "path" - "path/filepath" - "strings" "sync" - "github.com/gohugoio/hugo/helpers" - - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/cache/filecache" - - "github.com/BurntSushi/locker" -) - -const ( - CACHE_CLEAR_ALL = "clear_all" - CACHE_OTHER = "other" ) type ResourceCache struct { @@ -42,123 +30,30 @@ type ResourceCache struct { sync.RWMutex - // Either resource.Resource or resource.Resources. - cache map[string]interface{} + // Memory cache with either + // resource.Resource or resource.Resources. + cache memcache.Getter fileCache *filecache.Cache - - // Provides named resource locks. - nlocker *locker.Locker -} - -// ResourceCacheKey converts the filename into the format used in the resource -// cache. -func ResourceCacheKey(filename string) string { - filename = filepath.ToSlash(filename) - return path.Join(resourceKeyPartition(filename), filename) -} - -func resourceKeyPartition(filename string) string { - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") - if ext == "" { - ext = CACHE_OTHER - } - return ext -} - -// Commonly used aliases and directory names used for some types. -var extAliasKeywords = map[string][]string{ - "sass": []string{"scss"}, - "scss": []string{"sass"}, -} - -// ResourceKeyPartitions resolves a ordered slice of partitions that is -// used to do resource cache invalidations. -// -// We use the first directory path element and the extension, so: -// a/b.json => "a", "json" -// b.json => "json" -// -// For some of the extensions we will also map to closely related types, -// e.g. "scss" will also return "sass". -// -func ResourceKeyPartitions(filename string) []string { - var partitions []string - filename = glob.NormalizePath(filename) - dir, name := path.Split(filename) - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(name)), ".") - - if dir != "" { - partitions = append(partitions, strings.Split(dir, "/")[0]) - } - - if ext != "" { - partitions = append(partitions, ext) - } - - if aliases, found := extAliasKeywords[ext]; found { - partitions = append(partitions, aliases...) - } - - if len(partitions) == 0 { - partitions = []string{CACHE_OTHER} - } - - return helpers.UniqueStringsSorted(partitions) -} - -// ResourceKeyContainsAny returns whether the key is a member of any of the -// given partitions. -// -// This is used for resource cache invalidation. -func ResourceKeyContainsAny(key string, partitions []string) bool { - parts := strings.Split(key, "/") - for _, p1 := range partitions { - for _, p2 := range parts { - if p1 == p2 { - return true - } - } - } - return false } -func newResourceCache(rs *Spec) *ResourceCache { +func newResourceCache(rs *Spec, memCache *memcache.Cache) *ResourceCache { return &ResourceCache{ rs: rs, fileCache: rs.FileCaches.AssetsCache(), - cache: make(map[string]interface{}), - nlocker: locker.NewLocker(), + cache: memCache.GetOrCreatePartition("resources", memcache.ClearOnChange), } } -func (c *ResourceCache) clear() { - c.Lock() - defer c.Unlock() - - c.cache = make(map[string]interface{}) - c.nlocker = locker.NewLocker() -} - -func (c *ResourceCache) Contains(key string) bool { - key = c.cleanKey(filepath.ToSlash(key)) - _, found := c.get(key) - return found -} - -func (c *ResourceCache) cleanKey(key string) string { - return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") -} - -func (c *ResourceCache) get(key string) (interface{}, bool) { - c.RLock() - defer c.RUnlock() - r, found := c.cache[key] - return r, found -} - -func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, error)) (resource.Resource, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) +func (c *ResourceCache) GetOrCreate(key string, clearWhen memcache.ClearWhen, f func() (resource.Resource, error)) (resource.Resource, error) { + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + ClearWhen: clearWhen, + } + }) if r == nil || err != nil { return nil, err } @@ -166,43 +61,19 @@ func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, err } func (c *ResourceCache) GetOrCreateResources(key string, f func() (resource.Resources, error)) (resource.Resources, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + } + }) if r == nil || err != nil { return nil, err } return r.(resource.Resources), nil } -func (c *ResourceCache) getOrCreate(key string, f func() (interface{}, error)) (interface{}, error) { - key = c.cleanKey(key) - // First check in-memory cache. - r, found := c.get(key) - if found { - return r, nil - } - // This is a potentially long running operation, so get a named lock. - c.nlocker.Lock(key) - - // Double check in-memory cache. - r, found = c.get(key) - if found { - c.nlocker.Unlock(key) - return r, nil - } - - defer c.nlocker.Unlock(key) - - r, err := f() - if err != nil { - return nil, err - } - - c.set(key, r) - - return r, nil - -} - func (c *ResourceCache) getFilenames(key string) (string, string) { filenameMeta := key + ".json" filenameContent := key + ".content" @@ -255,43 +126,3 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) return fi, fc, err } - -func (c *ResourceCache) set(key string, r interface{}) { - c.Lock() - defer c.Unlock() - c.cache[key] = r -} - -func (c *ResourceCache) DeletePartitions(partitions ...string) { - partitionsSet := map[string]bool{ - // Always clear out the resources not matching any partition. - "other": true, - } - for _, p := range partitions { - partitionsSet[p] = true - } - - if partitionsSet[CACHE_CLEAR_ALL] { - c.clear() - return - } - - c.Lock() - defer c.Unlock() - - for k := range c.cache { - clear := false - for p := range partitionsSet { - if strings.Contains(k, p) { - // There will be some false positive, but that's fine. - clear = true - break - } - } - - if clear { - delete(c.cache, k) - } - } - -} diff --git a/resources/resource_cache_test.go b/resources/resource_cache_test.go deleted file mode 100644 index bcb24102594..00000000000 --- a/resources/resource_cache_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resources - -import ( - "path/filepath" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestResourceKeyPartitions(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - input string - expected []string - }{ - {"a.js", []string{"js"}}, - {"a.scss", []string{"sass", "scss"}}, - {"a.sass", []string{"sass", "scss"}}, - {"d/a.js", []string{"d", "js"}}, - {"js/a.js", []string{"js"}}, - {"D/a.JS", []string{"d", "js"}}, - {"d/a", []string{"d"}}, - {filepath.FromSlash("/d/a.js"), []string{"d", "js"}}, - {filepath.FromSlash("/d/e/a.js"), []string{"d", "js"}}, - } { - c.Assert(ResourceKeyPartitions(test.input), qt.DeepEquals, test.expected, qt.Commentf(test.input)) - } -} - -func TestResourceKeyContainsAny(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - key string - filename string - expected bool - }{ - {"styles/css", "asdf.css", true}, - {"styles/css", "styles/asdf.scss", true}, - {"js/foo.bar", "asdf.css", false}, - } { - c.Assert(ResourceKeyContainsAny(test.key, ResourceKeyPartitions(test.filename)), qt.Equals, test.expected) - } -} diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index 1ea92bea397..818dbaf5294 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -17,9 +17,10 @@ package bundler import ( "fmt" "io" - "path" "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" @@ -81,8 +82,7 @@ func (r *multiReadSeekCloser) Close() error { // Concat concatenates the list of Resource objects. func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resource, error) { - // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(targetPath, memcache.ClearOnRebuild, func() (resource.Resource, error) { var resolvedm media.Type // The given set of resources must be of the same Media Type. diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 4ac20d36e5a..49604ca0720 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -18,7 +18,8 @@ package create import ( "path" "path/filepath" - "strings" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/hugofs/glob" @@ -43,7 +44,7 @@ func New(rs *resources.Spec) *Client { // Get creates a new Resource by opening the given filename in the assets filesystem. func (c *Client) Get(filename string) (resource.Resource, error) { filename = filepath.Clean(filename) - return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(filename), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(filename), memcache.ClearOnChange, func() (resource.Resource, error) { return c.rs.New(resources.ResourceSourceDescriptor{ Fs: c.rs.BaseFs.Assets.Fs, LazyPublish: true, @@ -74,13 +75,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro name = "__match" } - pattern = glob.NormalizePath(pattern) - partitions := glob.FilterGlobParts(strings.Split(pattern, "/")) - if len(partitions) == 0 { - partitions = []string{resources.CACHE_OTHER} - } - key := path.Join(name, path.Join(partitions...)) - key = path.Join(key, pattern) + key := path.Join(name, glob.NormalizePath(pattern)) return c.rs.ResourceCache.GetOrCreateResources(key, func() (resource.Resources, error) { var res resource.Resources @@ -116,7 +111,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + r, err := c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(targetPath), memcache.ClearOnRebuild, func() (resource.Resource, error) { return c.rs.New( resources.ResourceSourceDescriptor{ Fs: c.rs.FileCaches.AssetsCache().Fs, @@ -128,4 +123,9 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro }) + if err == nil { + r.(resource.Staler).MarkStale() + } + return r, err + } diff --git a/resources/resource_metadata_test.go b/resources/resource_metadata_test.go index c79a5002121..0414127b92a 100644 --- a/resources/resource_metadata_test.go +++ b/resources/resource_metadata_test.go @@ -209,12 +209,12 @@ func TestAssignMetadata(t *testing.T) { }}, } { - foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) - logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) - foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) - logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) - foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) - logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) + foo2 = newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) + logo2 = newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) + foo1 = newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) + logo1 = newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) + foo3 = newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) + logo3 = newGenericResource(spec, nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) resources = resource.Resources{ foo2, diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 81eed2f0203..d2abdbe053f 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/config" @@ -46,6 +48,7 @@ import ( func NewSpec( s *helpers.PathSpec, fileCaches filecache.Caches, + memCache *memcache.Cache, incr identity.Incrementer, logger *loggers.Logger, errorHandler herrors.ErrorSender, @@ -89,11 +92,11 @@ func NewSpec( PostProcessResources: make(map[string]postpub.PostPublishedResource), imageCache: newImageCache( fileCaches.ImageCache(), - + memCache, s, )} - rs.ResourceCache = newResourceCache(rs) + rs.ResourceCache = newResourceCache(rs, memCache) return rs, nil @@ -129,57 +132,11 @@ func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { return r.newResourceFor(fd) } -func (r *Spec) CacheStats() string { - r.imageCache.mu.RLock() - defer r.imageCache.mu.RUnlock() - - s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) - - count := 0 - for k := range r.imageCache.store { - if count > 5 { - break - } - s += "\n" + k - count++ - } - - return s -} - -func (r *Spec) ClearCaches() { - r.imageCache.clear() - r.ResourceCache.clear() -} - -func (r *Spec) DeleteBySubstring(s string) { - r.imageCache.deleteIfContains(s) -} - func (s *Spec) String() string { return "spec" } // TODO(bep) clean up below -func (r *Spec) newGenericResource(sourceFs afero.Fs, - targetPathBuilder func() page.TargetPaths, - osFileInfo os.FileInfo, - sourceFilename, - baseFilename string, - mediaType media.Type) *genericResource { - return r.newGenericResourceWithBase( - sourceFs, - nil, - nil, - targetPathBuilder, - osFileInfo, - sourceFilename, - baseFilename, - mediaType, - ) - -} - func (r *Spec) newGenericResourceWithBase( sourceFs afero.Fs, openReadSeekerCloser resource.OpenReadSeekCloser, diff --git a/resources/resource_test.go b/resources/resource_test.go index 7a0b8069d79..75ac94f79ce 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -34,7 +34,7 @@ func TestGenericResource(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) - r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo.css") @@ -48,11 +48,11 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := newTargetPaths("/foo") - r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo/foo.css") - c.Assert(r.Key(), qt.Equals, "/foo/foo.css") + c.Assert(r.Key(), qt.Equals, "foo/foo.css") c.Assert(r.ResourceType(), qt.Equals, "css") } @@ -105,10 +105,10 @@ func TestResourcesByType(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), - spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo.png", "logo.css", pngType), + newGenericResource(spec, nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} c.Assert(len(resources.ByType("css")), qt.Equals, 3) c.Assert(len(resources.ByType("image")), qt.Equals, 1) @@ -119,11 +119,11 @@ func TestResourcesGetByPrefix(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} c.Assert(resources.GetMatch("asdf*"), qt.IsNil) c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -148,14 +148,14 @@ func TestResourcesGetMatch(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), } c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -212,7 +212,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := resource.Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)} + resources := resource.Resources{newGenericResource(spec, nil, nil, nil, "/a/"+a100, a100, media.CSSType)} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -228,17 +228,17 @@ func benchResources(b *testing.B) resource.Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) } return resources @@ -266,7 +266,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } b.StartTimer() diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 8eacf7da4e6..3d7af38c23d 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -16,6 +16,8 @@ package htesting import ( "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -51,7 +53,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 0462f7ecdfc..3f5d6905f06 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -4,6 +4,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/cache/memcache" + "image" "io" "io/ioutil" @@ -90,7 +92,9 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + mc := memcache.New(memcache.Config{}) + + spec, err := NewSpec(s, filecaches, mc, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -129,7 +133,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir @@ -207,3 +211,22 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { t.Fatalf("Failed to write file: %s", err) } } + +func newGenericResource(r *Spec, sourceFs afero.Fs, + targetPathBuilder func() page.TargetPaths, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + nil, + nil, + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} diff --git a/resources/transform.go b/resources/transform.go index 98aee3c2a6f..5653b42b008 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -18,9 +18,12 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/resources/images/exif" @@ -42,6 +45,8 @@ var ( _ resource.ContentResource = (*resourceAdapter)(nil) _ resource.ReadSeekCloserResource = (*resourceAdapter)(nil) _ resource.Resource = (*resourceAdapter)(nil) + _ resource.Staler = (*resourceAdapter)(nil) + _ resource.Staler = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) _ resource.ResourceMetaProvider = (*resourceAdapter)(nil) @@ -60,12 +65,17 @@ func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResour if lazyPublish { po = &publishOnce{} } + + s := &resourceOrigin{} + return &resourceAdapter{ resourceTransformations: &resourceTransformations{}, + resourceOrigin: s, resourceAdapterInner: &resourceAdapterInner{ - spec: spec, - publishOnce: po, - target: target, + spec: spec, + resourceOrigin: s, + publishOnce: po, + target: target, }, } } @@ -150,6 +160,11 @@ type publishOnce struct { type resourceAdapter struct { commonResource + + // This state is carried over into any clone of this adapter + // (when passed through a Hugo pipe). + *resourceOrigin + *resourceTransformations *resourceAdapterInner } @@ -188,8 +203,7 @@ func (r *resourceAdapter) Exif() (*exif.Exif, error) { } func (r *resourceAdapter) Key() string { - r.init(false, false) - return r.target.(resource.Identifier).Key() + return r.TransformationKey() } func (r *resourceAdapter) MediaType() media.Type { @@ -252,9 +266,10 @@ func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransfo } r.resourceAdapterInner = &resourceAdapterInner{ - spec: r.spec, - publishOnce: &publishOnce{}, - target: r.target, + spec: r.spec, + resourceOrigin: r.resourceOrigin, + publishOnce: &publishOnce{}, + target: r.target, } return &r, nil @@ -297,6 +312,57 @@ func (r *resourceAdapter) publish() { } func (r *resourceAdapter) TransformationKey() string { + r.transformationsKeyInit.Do(func() { + if len(r.transformations) == 0 { + r.transformationsKey = r.target.Key() + return + } + + var adder string + for _, tr := range r.transformations { + adder = adder + "_" + tr.Key().Value() + } + + key := r.target.Key() + adder = "_" + helpers.MD5String(adder) + + // Preserve any file extension if possible. + dotIdx := strings.LastIndex(key, ".") + if dotIdx == -1 { + key += adder + } else { + key = key[:dotIdx] + adder + key[dotIdx:] + } + + key = memcache.CleanKey(key) + r.transformationsKey = key + }) + + return r.transformationsKey +} + +// We changed the format of the resource cache keys in Hugo v0.75. +// To reduce the nois, especially on the theme site, we fall back to reading +// files on the old format. +// TODO(bep) eventually remove. +func (r *resourceAdapter) transformationKeyV074() string { + cleanKey := func(key string) string { + return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") + } + + resourceKeyPartition := func(filename string) string { + ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") + if ext == "" { + ext = "other" + } + return ext + } + + resourceCacheKey := func(filename string) string { + filename = filepath.ToSlash(filename) + return path.Join(resourceKeyPartition(filename), filename) + } + // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. var key string @@ -304,35 +370,33 @@ func (r *resourceAdapter) TransformationKey() string { key = key + "_" + tr.Key().Value() } - base := ResourceCacheKey(r.target.Key()) - return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key) -} + base := resourceCacheKey(r.target.RelPermalink()) + return cleanKey(base) + "_" + helpers.MD5String(key) -func (r *resourceAdapter) transform(publish, setContent bool) error { - cache := r.spec.ResourceCache +} +func (r *resourceAdapter) getOrTransform(publish, setContent bool) error { key := r.TransformationKey() + res, err := r.spec.ResourceCache.cache.GetOrCreate(key, func() memcache.Entry { + r, err := r.transform(key, publish, setContent) + return memcache.Entry{ + Value: r, + Err: err, + } + }) - cached, found := cache.get(key) - - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - return nil + if err != nil { + return err } - // Acquire a write lock for the named transformation. - cache.nlocker.Lock(key) - // Check the cache again. - cached, found = cache.get(key) - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - cache.nlocker.Unlock(key) - return nil - } + r.resourceAdapterInner = res.(*resourceAdapterInner) - defer cache.nlocker.Unlock(key) - defer cache.set(key, r.resourceAdapterInner) + return nil +} + +func (r *resourceAdapter) transform(key string, publish, setContent bool) (*resourceAdapterInner, error) { + cache := r.spec.ResourceCache b1 := bp.GetBuffer() b2 := bp.GetBuffer() defer bp.PutBuffer(b1) @@ -353,7 +417,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { contentrc, err := contentReadSeekerCloser(r.target) if err != nil { - return err + return nil, err } defer contentrc.Close() @@ -426,21 +490,25 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { } else { err = tr.Transform(tctx) if err != nil && err != herrors.ErrFeatureNotAvailable { - return newErr(err) + return nil, newErr(err) } if mayBeCachedOnDisk { tryFileCache = r.spec.BuildConfig.UseResourceCache(err) } if err != nil && !tryFileCache { - return newErr(err) + return nil, newErr(err) } } if tryFileCache { f := r.target.tryTransformedFileCache(key, updates) if f == nil { - return newErr(errors.Errorf("resource %q not found in file cache", key)) + keyOldFormat := r.transformationKeyV074() + f = r.target.tryTransformedFileCache(keyOldFormat, updates) + if f == nil { + return nil, newErr(errors.Errorf("resource %q not found in file cache", key)) + } } transformedContentr = f updates.sourceFs = cache.fileCache.Fs @@ -465,7 +533,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { if publish { publicw, err := r.target.openPublishFileForWriting(updates.targetPath) if err != nil { - return err + return nil, err } publishwriters = append(publishwriters, publicw) } @@ -475,7 +543,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { // Also write it to the cache fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata()) if err != nil { - return err + return nil, err } updates.sourceFilename = &fi.Name updates.sourceFs = cache.fileCache.Fs @@ -506,7 +574,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { publishw := hugio.NewMultiWriteCloser(publishwriters...) _, err = io.Copy(publishw, transformedContentr) if err != nil { - return err + return nil, err } publishw.Close() @@ -517,11 +585,11 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { newTarget, err := r.target.cloneWithUpdates(updates) if err != nil { - return err + return nil, err } r.target = newTarget - return nil + return r.resourceAdapterInner, nil } func (r *resourceAdapter) init(publish, setContent bool) { @@ -541,7 +609,7 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { r.publishOnce = nil } - r.transformationsErr = r.transform(publish, setContent) + r.transformationsErr = r.getOrTransform(publish, setContent) if r.transformationsErr != nil { if r.spec.ErrorSender != nil { r.spec.ErrorSender.SendError(r.transformationsErr) @@ -559,16 +627,24 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { type resourceAdapterInner struct { target transformableResource + *resourceOrigin + spec *Spec // Handles publishing (to /public) if needed. *publishOnce } +func (r *resourceAdapterInner) ResourceTarget() resource.Resource { + return r.target +} + type resourceTransformations struct { - transformationsInit sync.Once - transformationsErr error - transformations []ResourceTransformation + transformationsInit sync.Once + transformationsErr error + transformationsKeyInit sync.Once + transformationsKey string + transformations []ResourceTransformation } type transformableResource interface { @@ -603,7 +679,6 @@ func (u *transformationUpdate) toTransformedResourceMetadata() transformedResour } func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) { - u.targetPath = ctx.OutPath u.mediaType = ctx.OutMediaType u.data = ctx.Data u.targetPath = ctx.InPath diff --git a/tpl/openapi/openapi3/openapi3.go b/tpl/openapi/openapi3/openapi3.go index 7dfd2f6a797..a95e288e1a4 100644 --- a/tpl/openapi/openapi3/openapi3.go +++ b/tpl/openapi/openapi3/openapi3.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" kopenapi3 "github.com/getkin/kin-openapi/openapi3" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/resources/resource" @@ -29,22 +29,15 @@ import ( // New returns a new instance of the openapi3-namespaced template functions. func New(deps *deps.Deps) *Namespace { - // TODO1 consolidate when merging that "other branch" -- but be aware of the keys. - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - return &Namespace{ - cache: cache, + cache: deps.MemCache.GetOrCreatePartition("tpl/openapi3", memcache.ClearOnChange), deps: deps, } } // Namespace provides template functions for the "openapi3". type Namespace struct { - cache *namedmemcache.Cache + cache memcache.Getter deps *deps.Deps } @@ -55,21 +48,22 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg return nil, errors.New("no Key set in Resource") } - v, err := ns.cache.GetOrCreate(key, func() (interface{}, error) { + v, err := ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } + defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } s := &kopenapi3.Swagger{} @@ -80,12 +74,14 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg err = metadecoders.Default.UnmarshalTo(b, f, s) } if err != nil { - return nil, err + return memcache.Entry{Err: err} } err = kopenapi3.NewSwaggerLoader().ResolveRefsIn(s, nil) - return s, err + return memcache.Entry{Value: s, Err: err, StaleFunc: func() bool { + return resource.IsStaleAny(r) + }} }) if err != nil { diff --git a/tpl/transform/init_test.go b/tpl/transform/init_test.go index 47bd8a391e6..393b6b140d8 100644 --- a/tpl/transform/init_test.go +++ b/tpl/transform/init_test.go @@ -16,6 +16,8 @@ package transform import ( "testing" + "github.com/gohugoio/hugo/cache/memcache" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/htesting/hqt" @@ -28,7 +30,9 @@ func TestInit(t *testing.T) { var ns *internal.TemplateFuncsNamespace for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns = nsf(&deps.Deps{}) + ns = nsf(&deps.Deps{ + MemCache: memcache.New(memcache.Config{}), + }) if ns.Name == name { found = true break diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index b168d2a50d4..e4ea17beba2 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -18,7 +18,7 @@ import ( "html" "html/template" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" @@ -27,22 +27,19 @@ import ( // New returns a new instance of the transform-namespaced template functions. func New(deps *deps.Deps) *Namespace { - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - + if deps.MemCache == nil { + panic("must provide MemCache") + } return &Namespace{ - cache: cache, deps: deps, + cache: deps.MemCache.GetOrCreatePartition("tpl/transform", memcache.ClearOnChange), } } // Namespace provides template functions for the "transform" namespace. type Namespace struct { - cache *namedmemcache.Cache deps *deps.Deps + cache memcache.Getter } // Emojify returns a copy of s with all emoji codes replaced with actual emojis. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index b3f4206ff6b..1a14661f246 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -16,6 +16,8 @@ package transform import ( "html/template" + "github.com/gohugoio/hugo/cache/memcache" + "testing" "github.com/gohugoio/hugo/common/loggers" @@ -251,6 +253,7 @@ func newDeps(cfg config.Provider) *deps.Deps { return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), + MemCache: memcache.New(memcache.Config{}), ContentSpec: cs, } } diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index b606c870aa7..ea43f6d0ff9 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -17,6 +17,8 @@ import ( "io/ioutil" "strings" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/common/types" @@ -69,24 +71,28 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key += decoder.OptionsKey() } - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } - return decoder.Unmarshal(b, f) + v, err := decoder.Unmarshal(b, f) + + return memcache.Entry{Value: v, Err: err, StaleFunc: func() bool { + return resource.IsStaleAny(r) + }} }) } @@ -97,13 +103,15 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key := helpers.MD5String(dataStr) - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := decoder.FormatFromContentString(dataStr) if f == "" { - return nil, errors.New("unknown format") + return memcache.Entry{Err: errors.New("unknown format")} } - return decoder.Unmarshal([]byte(dataStr), f) + v, err := decoder.Unmarshal([]byte(dataStr), f) + + return memcache.Entry{Value: v, Err: err} }) } diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 183bdefd5bc..8796cef3724 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -145,7 +145,7 @@ a;b;c`, mime: media.CSVType}, map[string]interface{}{"DElimiter": ";", "Comment" {tstNoStringer{}, nil, false}, } { - ns.cache.Clear() + ns.deps.MemCache.Clear() var args []interface{}