From 8a38818195c9f3aa5121b0abed615c7de3952008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 25 Jun 2020 11:32:42 +0200 Subject: [PATCH] Add central LRU cache that adjusts to available memory Hugo has always been a active user of in-memory caches, but before this commit we did nothing to control the memory usage. One failing example would be loading lots of big JSON data files and unmarshal them via `transform.Unmarshal`. This commit consolidates all these caches into one single LRU cache with an eviction strategy that also considers used vs. available memory. Hugo will try to limit its memory usage to 1/4 or total system memory, but this can be controlled with the `HUGO_MEMORYLIMIT` environment variable (a float value representing Gigabytes). A natural next step after this would be to use this cache for `.Content`. Fixes #7425 Fixes #7437 Fixes #7436 Updates #7544 --- cache/memcache/memcache.go | 460 ++++++++++++++++++ cache/memcache/memcache_test.go | 169 +++++++ cache/namedmemcache/named_cache.go | 79 --- cache/namedmemcache/named_cache_test.go | 80 --- commands/hugo.go | 22 +- config/env.go | 35 ++ config/env_test.go | 2 + deps/deps.go | 20 +- .../en/getting-started/configuration.md | 9 + go.mod | 2 + go.sum | 19 + helpers/general.go | 18 + hugofs/fileinfo.go | 5 + hugofs/rootmapping_fs.go | 7 +- hugolib/content_map_page.go | 7 + hugolib/filesystems/basefs.go | 61 ++- hugolib/hugo_sites_build.go | 4 + hugolib/hugo_sites_build_test.go | 81 ++- hugolib/hugo_sites_multihost_test.go | 2 +- hugolib/pagebundler_test.go | 4 +- hugolib/pages_capture.go | 4 - hugolib/paths/paths.go | 6 +- hugolib/resource_chain_test.go | 2 +- hugolib/resource_change_test.go | 322 ++++++++++++ hugolib/site.go | 16 +- hugolib/testhelpers_test.go | 27 +- modules/client.go | 2 +- resources/image.go | 4 +- resources/image_cache.go | 166 +++---- resources/image_test.go | 2 +- resources/resource.go | 52 +- resources/resource/resourcetypes.go | 30 ++ resources/resource_cache.go | 213 +------- resources/resource_cache_test.go | 58 --- .../resource_factories/bundler/bundler.go | 6 +- resources/resource_factories/create/create.go | 20 +- resources/resource_metadata_test.go | 12 +- resources/resource_spec.go | 53 +- resources/resource_test.go | 50 +- .../htesting/testhelpers.go | 4 +- resources/testhelpers_test.go | 27 +- resources/transform.go | 161 ++++-- tpl/openapi/openapi3/openapi3.go | 28 +- tpl/transform/init_test.go | 6 +- tpl/transform/transform.go | 15 +- tpl/transform/transform_test.go | 3 + tpl/transform/unmarshal.go | 24 +- tpl/transform/unmarshal_test.go | 2 +- 48 files changed, 1654 insertions(+), 747 deletions(-) create mode 100644 cache/memcache/memcache.go create mode 100644 cache/memcache/memcache_test.go delete mode 100644 cache/namedmemcache/named_cache.go delete mode 100644 cache/namedmemcache/named_cache_test.go create mode 100644 hugolib/resource_change_test.go delete mode 100644 resources/resource_cache_test.go diff --git a/cache/memcache/memcache.go b/cache/memcache/memcache.go new file mode 100644 index 00000000000..7cfd363edac --- /dev/null +++ b/cache/memcache/memcache.go @@ -0,0 +1,460 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memcache provides the core memory cache used in Hugo. +package memcache + +import ( + "math" + "path" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/resources/resource" + + "github.com/gohugoio/hugo/helpers" + "github.com/pkg/errors" + + "github.com/BurntSushi/locker" + "github.com/karlseguin/ccache/v2" +) + +const ( + gigabyte = 1 << 30 +) + +const ( + ClearOnRebuild ClearWhen = iota + ClearOnChange + ClearNever +) + +type Getter interface { + GetOrCreate(path string, create func() Entry) (interface{}, error) + Clear() +} + +var errInvalidKey = errors.New("invalid cache key") + +// New creates a new cache. +func New(conf Config) *Cache { + if conf.TTL == 0 { + conf.TTL = time.Second * 33 + } + if conf.CheckInterval == 0 { + conf.CheckInterval = time.Second * 2 + } + if conf.MaxSize == 0 { + conf.MaxSize = 100000 + } + if conf.ItemsToPrune == 0 { + conf.ItemsToPrune = 200 + } + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + stats := &stats{ + memstatsStart: m, + maxSize: conf.MaxSize, + availableMemory: config.GetMemoryLimit(), + } + + conf.MaxSize = stats.adjustMaxSize(nil) + + c := &Cache{ + conf: conf, + cache: ccache.Layered(ccache.Configure().MaxSize(conf.MaxSize).ItemsToPrune(conf.ItemsToPrune)), + getters: make(map[string]*getter), + ttl: conf.TTL, + stats: stats, + nlocker: locker.NewLocker(), + } + + c.stop = c.start() + + return c +} + +// Cache configures a cache. +type Cache struct { + mu sync.Mutex + getters map[string]*getter + + conf Config + cache *ccache.LayeredCache + + ttl time.Duration + nlocker *locker.Locker + + stats *stats + stopOnce sync.Once + stop func() +} + +var extAliasKeywords = map[string]map[string]bool{ + "sass": map[string]bool{"scss": true}, + "js": map[string]bool{"ts": true}, +} + +func init() { + for k, v := range extAliasKeywords { + for vv, _ := range v { + if _, found := extAliasKeywords[vv]; !found { + m := make(map[string]bool) + m[k] = true + for vv2, _ := range v { + if vv2 != vv { + m[vv2] = true + } + } + extAliasKeywords[vv] = m + } + } + } + +} + +// ClearOn clears all the caches with that clearing strategy. +// This method is not thread safe. +func (c *Cache) ClearOn(when ClearWhen, changeset ...string) { + for _, g := range c.getters { + if g.clearWhen == ClearNever { + continue + } + + if g.clearWhen == when { + // Clear all. + g.Clear() + return + } + + changeset = helpers.UniqueStrings(changeset) + shouldDelete := func(key string, e Entry) bool { + if e.ClearWhen == when || resource.IsStaleAny(e, e.Value) { + return true + } + + if len(changeset) == 0 { + return false + } + + for _, filename := range changeset { + path := CleanKey(filename) + prefix, suffix := splitBasePathAndExt(path) + + // Will clear out all files that share a common root path, + // e.g "styles" in "styles/main.css". + if strings.HasPrefix(key, prefix) { + return true + } + + switch v := e.Value.(type) { + case resource.MediaTypeProvider: + // This will have some potential false positives, + // i.e. more work done, but this is supposed to + // evict entries for dependencies that Hugo isn't in fully + // control of, e.g.: + // A change in /assets/bootstrap/somefile.scss will trigger + // a rebuild of /assets/scss/main.scss. + checkSuffix := func(s string) bool { + return helpers.InStringArray(v.MediaType().Suffixes, s) + } + + if checkSuffix(suffix) { + return true + } + if aliases, found := extAliasKeywords[suffix]; found { + for alias, _ := range aliases { + if checkSuffix(alias) { + return true + } + } + } + } + } + + return false + } + + g.c.cache.DeleteFunc(g.partition, func(key string, item *ccache.Item) bool { + e := item.Value().(Entry) + if shouldDelete(key, e) { + resource.MarkStale(e.Value) + return true + } + return false + }) + + } +} + +// Clear clears the cache state. +// This method is not thread safe. +func (c *Cache) Clear() { + c.nlocker = locker.NewLocker() + c.cache.Clear() +} + +func (c *Cache) DeleteAll(primary string) bool { + return c.cache.DeleteAll(primary) +} + +func (c *Cache) GetDropped() int { + return c.cache.GetDropped() +} + +func (c *Cache) GetOrCreatePartition(partition string, clearWhen ClearWhen) Getter { + c.mu.Lock() + defer c.mu.Unlock() + + g, found := c.getters[partition] + if found { + if g.clearWhen != clearWhen { + panic("GetOrCreatePartition called with the same partition but different clearing strategy.") + } + return g + } + + g = &getter{ + partition: partition, + c: c, + clearWhen: clearWhen, + } + + c.getters[partition] = g + + return g +} + +func (c *Cache) Stop() { + c.stopOnce.Do(func() { + c.stop() + c.cache.Stop() + }) +} + +func (c *Cache) start() func() { + ticker := time.NewTicker(c.conf.CheckInterval) + quit := make(chan struct{}) + + checkAndAdjustMaxSize := func() { + var m runtime.MemStats + cacheDropped := c.GetDropped() + c.stats.decr(cacheDropped) + + runtime.ReadMemStats(&m) + c.stats.memstatsCurrent = m + c.stats.adjustMaxSize(c.cache.SetMaxSize) + + //fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) + + } + go func() { + for { + select { + case <-ticker.C: + checkAndAdjustMaxSize() + case <-quit: + ticker.Stop() + return + } + } + }() + + return func() { + close(quit) + } +} + +func (c *Cache) get(primary, secondary string) (interface{}, bool) { + v := c.cache.Get(primary, secondary) + if v == nil { + return nil, false + } + + return v.Value(), true +} + +// GetOrCreate tries to get the value with the given cache paths, if not found +// create will be called and cached. +// This method is thread safe. +func (c *Cache) getOrCreate(primary, secondary string, create func() Entry) (interface{}, error) { + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // The provided create function may be a relatively time consuming operation, + // and there will in the commmon case be concurrent requests for the same key'd + // resource, so make sure we pause these until the result is ready. + path := primary + secondary + c.nlocker.Lock(path) + defer c.nlocker.Unlock(path) + + // Try again. + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // Create it and store it in cache. + entry := create() + entry.size = 1 // For now. + if entry.Err != nil { + entry.ClearWhen = ClearOnRebuild + } + + c.cache.Set(primary, secondary, entry, c.ttl) + c.stats.incr(1) + + return entry.Value, entry.Err +} + +type ClearWhen int + +type Config struct { + CheckInterval time.Duration + MaxSize int64 + ItemsToPrune uint32 + TTL time.Duration + Running bool +} + +type Entry struct { + Value interface{} + size int64 + Err error + StaleFunc func() bool + ClearWhen +} + +func (e Entry) Size() int64 { + return e.size +} + +func (e Entry) IsStale() bool { + return e.StaleFunc != nil && e.StaleFunc() +} + +type getter struct { + c *Cache + partition string + + clearWhen ClearWhen +} + +func (g *getter) Clear() { + g.c.DeleteAll(g.partition) +} + +func (g *getter) GetOrCreate(path string, create func() Entry) (interface{}, error) { + return g.c.getOrCreate(g.partition, path, create) +} + +type stats struct { + memstatsStart runtime.MemStats + memstatsCurrent runtime.MemStats + maxSize int64 + availableMemory uint64 + numItems uint64 +} + +func (s *stats) resizeFactor() float64 { + if s.memstatsCurrent.Alloc == 0 { + return 1.0 + } + return math.Floor(float64(s.availableMemory/s.memstatsCurrent.Alloc)*10) / 10 +} + +func (s *stats) adjustMaxSize(setter func(size int64)) int64 { + newSize := int64(float64(s.maxSize) * s.resizeFactor()) + if newSize != s.maxSize && setter != nil { + setter(newSize) + } + return newSize +} + +func (s *stats) decr(i int) { + atomic.AddUint64(&s.numItems, ^uint64(i-1)) +} + +func (s *stats) incr(i int) { + atomic.AddUint64(&s.numItems, uint64(i)) +} + +// This assumes a valid key path. +func splitBasePathAndExt(path string) (string, string) { + dotIdx := strings.LastIndex(path, ".") + ext := path[dotIdx+1:] + slashIdx := strings.Index(path, "/") + + return path[:slashIdx], ext +} + +// CleanKey turns s into a format suitable for a cache key for this package. +// The key will be a Unix-styled path without any leading slash. +// If the input string does not contain any slash, a root will be prepended. +// If the input string does not contain any ".", a dummy file suffix will be appended. +// These are to make sure that they can effectively partake in the "cache cleaning" +// strategy used in server mode. +func CleanKey(s string) string { + s = path.Clean(helpers.ToSlashTrimLeading(s)) + if !strings.ContainsRune(s, '/') { + s = "_root/" + s + } + if !strings.ContainsRune(s, '.') { + s += "." + unknownExtension + } + + return s +} + +// InsertKeyPathElement inserts the given element after the first '/' in key. +func InsertKeyPathElements(key string, elements ...string) string { + slashIdx := strings.Index(key, "/") + return key[:slashIdx] + "/" + path.Join(elements...) + key[slashIdx:] +} + +const unknownExtension = "unkn" + +func keyValid(s string) bool { + if len(s) < 5 { + return false + } + if strings.ContainsRune(s, '\\') { + return false + } + if strings.HasPrefix(s, "/") { + return false + } + if !strings.ContainsRune(s, '/') { + return false + } + + dotIdx := strings.Index(s, ".") + if dotIdx == -1 || dotIdx == len(s)-1 { + return false + } + + return true +} diff --git a/cache/memcache/memcache_test.go b/cache/memcache/memcache_test.go new file mode 100644 index 00000000000..baad1e17222 --- /dev/null +++ b/cache/memcache/memcache_test.go @@ -0,0 +1,169 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memcache + +import ( + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + qt "github.com/frankban/quicktest" +) + +func TestCache(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{}) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + for i := 0; i < 5; i++ { + v1, err := cache.getOrCreate("a", "a1", create) + c.Assert(err, qt.IsNil) + c.Assert(v1, qt.Equals, 1) + v2, err := cache.getOrCreate("a", "a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v2, qt.Equals, 2) + } + + cache.Clear() + + v3, err := cache.getOrCreate("a", "a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v3, qt.Equals, 3) +} + +func TestCacheConcurrent(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + var wg sync.WaitGroup + + cache := New(Config{}) + + create := func(i int) func() Entry { + return func() Entry { + return Entry{Value: i} + } + } + + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + id := fmt.Sprintf("id%d", j) + v, err := cache.getOrCreate("a", id, create(j)) + c.Assert(err, qt.IsNil) + c.Assert(v, qt.Equals, j) + } + }() + } + wg.Wait() +} + +func TestCacheMemStats(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{ + ItemsToPrune: 10, + CheckInterval: 500 * time.Millisecond, + }) + + s := cache.stats + + c.Assert(s.memstatsStart.Alloc > 0, qt.Equals, true) + c.Assert(s.memstatsCurrent.Alloc, qt.Equals, uint64(0)) + c.Assert(s.availableMemory > 0, qt.Equals, true) + c.Assert(s.numItems, qt.Equals, uint64(0)) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + for i := 1; i <= 20; i++ { + _, err := cache.getOrCreate("a", fmt.Sprintf("b%d", i), create) + c.Assert(err, qt.IsNil) + } + + c.Assert(s.numItems, qt.Equals, uint64(20)) + cache.cache.SetMaxSize(10) + time.Sleep(time.Millisecond * 600) + c.Assert(int(s.numItems), qt.Equals, 10) + + c.Assert(s.memstatsCurrent.Alloc > 0, qt.Equals, true) +} + +func TestSplitBasePathAndExt(t *testing.T) { + t.Parallel() + c := qt.New(t) + + tests := []struct { + path string + a string + b string + }{ + {"a/b.json", "a", "json"}, + {"a/b/c/d.json", "a", "json"}, + } + for i, this := range tests { + msg := qt.Commentf("test %d", i) + a, b := splitBasePathAndExt(this.path) + + c.Assert(a, qt.Equals, this.a, msg) + c.Assert(b, qt.Equals, this.b, msg) + } + +} + +func TestCleanKey(t *testing.T) { + c := qt.New(t) + + c.Assert(CleanKey(filepath.FromSlash("a/b/c.js")), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a//b////c.js"), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a.js"), qt.Equals, "_root/a.js") + c.Assert(CleanKey("b/a"), qt.Equals, "b/a.unkn") + +} + +func TestKeyValid(t *testing.T) { + c := qt.New(t) + + c.Assert(keyValid("a/b.j"), qt.Equals, true) + c.Assert(keyValid("a/b."), qt.Equals, false) + c.Assert(keyValid("a/b"), qt.Equals, false) + c.Assert(keyValid("/a/b.txt"), qt.Equals, false) + c.Assert(keyValid("a\\b.js"), qt.Equals, false) + +} + +func TestInsertKeyPathElement(t *testing.T) { + c := qt.New(t) + + c.Assert(InsertKeyPathElements("a/b.j", "en"), qt.Equals, "a/en/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "en", "foo"), qt.Equals, "a/en/foo/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "", "foo"), qt.Equals, "a/foo/b.j") + +} diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go deleted file mode 100644 index d8c229a013b..00000000000 --- a/cache/namedmemcache/named_cache.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package namedmemcache provides a memory cache with a named lock. This is suitable -// for situations where creating the cached resource can be time consuming or otherwise -// resource hungry, or in situations where a "once only per key" is a requirement. -package namedmemcache - -import ( - "sync" - - "github.com/BurntSushi/locker" -) - -// Cache holds the cached values. -type Cache struct { - nlocker *locker.Locker - cache map[string]cacheEntry - mu sync.RWMutex -} - -type cacheEntry struct { - value interface{} - err error -} - -// New creates a new cache. -func New() *Cache { - return &Cache{ - nlocker: locker.NewLocker(), - cache: make(map[string]cacheEntry), - } -} - -// Clear clears the cache state. -func (c *Cache) Clear() { - c.mu.Lock() - defer c.mu.Unlock() - - c.cache = make(map[string]cacheEntry) - c.nlocker = locker.NewLocker() - -} - -// GetOrCreate tries to get the value with the given cache key, if not found -// create will be called and cached. -// This method is thread safe. It also guarantees that the create func for a given -// key is invoced only once for this cache. -func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) { - c.mu.RLock() - entry, found := c.cache[key] - c.mu.RUnlock() - - if found { - return entry.value, entry.err - } - - c.nlocker.Lock(key) - defer c.nlocker.Unlock(key) - - // Create it. - value, err := create() - - c.mu.Lock() - c.cache[key] = cacheEntry{value: value, err: err} - c.mu.Unlock() - - return value, err -} diff --git a/cache/namedmemcache/named_cache_test.go b/cache/namedmemcache/named_cache_test.go deleted file mode 100644 index 9feddb11f2a..00000000000 --- a/cache/namedmemcache/named_cache_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package namedmemcache - -import ( - "fmt" - "sync" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestNamedCache(t *testing.T) { - t.Parallel() - c := qt.New(t) - - cache := New() - - counter := 0 - create := func() (interface{}, error) { - counter++ - return counter, nil - } - - for i := 0; i < 5; i++ { - v1, err := cache.GetOrCreate("a1", create) - c.Assert(err, qt.IsNil) - c.Assert(v1, qt.Equals, 1) - v2, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v2, qt.Equals, 2) - } - - cache.Clear() - - v3, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v3, qt.Equals, 3) -} - -func TestNamedCacheConcurrent(t *testing.T) { - t.Parallel() - - c := qt.New(t) - - var wg sync.WaitGroup - - cache := New() - - create := func(i int) func() (interface{}, error) { - return func() (interface{}, error) { - return i, nil - } - } - - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - id := fmt.Sprintf("id%d", j) - v, err := cache.GetOrCreate(id, create(j)) - c.Assert(err, qt.IsNil) - c.Assert(v, qt.Equals, j) - } - }() - } - wg.Wait() -} diff --git a/commands/hugo.go b/commands/hugo.go index 5442c32d708..de311aaa646 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -433,8 +433,14 @@ func (c *commandeer) initMemTicker() func() { quit := make(chan struct{}) printMem := func() { var m runtime.MemStats + var cacheDropped int + h := c.hugo() + if h != nil && h.MemCache != nil { + cacheDropped = h.MemCache.GetDropped() + } + runtime.ReadMemStats(&m) - fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\n\n", formatByteCount(m.Alloc), formatByteCount(m.TotalAlloc), formatByteCount(m.Sys), m.NumGC) + fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) } @@ -1215,17 +1221,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { return name } - -func formatByteCount(b uint64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", - float64(b)/float64(div), "kMGTPE"[exp]) -} diff --git a/config/env.go b/config/env.go index f482cd24733..06233c4b641 100644 --- a/config/env.go +++ b/config/env.go @@ -17,6 +17,9 @@ import ( "os" "runtime" "strconv" + + "github.com/pbnjay/memory" + "strings" ) @@ -33,6 +36,38 @@ func GetNumWorkerMultiplier() int { return runtime.NumCPU() } +const ( + gigabyte = 1 << 30 +) + +// GetMemoryLimit returns the upper memory limit in bytes for Hugo's in-memory caches. +// Note that this does not represent "all of the memory" that Hugo will use, +// so it needs to be set to a lower number than the available system memory. +// It will read from the HUGO_MEMORYLIMIT (in Gigabytes) environment variable. +// If that is not set, it will set aside a quarter of the total system memory. +func GetMemoryLimit() uint64 { + if mem := os.Getenv("HUGO_MEMORYLIMIT"); mem != "" { + if v := stringToGibabyte(mem); v > 0 { + return v + } + + } + + m := memory.TotalMemory() + if m != 0 { + return uint64(m / 4) + } + + return 2 * gigabyte +} + +func stringToGibabyte(f string) uint64 { + if v, err := strconv.ParseFloat(f, 32); err == nil && v > 0 { + return uint64(v * gigabyte) + } + return 0 +} + // SetEnvVars sets vars on the form key=value in the oldVars slice. func SetEnvVars(oldVars *[]string, keyValues ...string) { for i := 0; i < len(keyValues); i += 2 { diff --git a/config/env_test.go b/config/env_test.go index 3c402b9ef8d..3b0faac3b7d 100644 --- a/config/env_test.go +++ b/config/env_test.go @@ -29,4 +29,6 @@ func TestSetEnvVars(t *testing.T) { key, val := SplitEnvVar("HUGO=rocks") c.Assert(key, qt.Equals, "HUGO") c.Assert(val, qt.Equals, "rocks") + + c.Assert(stringToGibabyte("1.0"), qt.Equals, uint64(1073741824)) } diff --git a/deps/deps.go b/deps/deps.go index 82a16ba5947..c7ef99b65f5 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -5,6 +5,8 @@ import ( "sync/atomic" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/cache/filecache" @@ -62,9 +64,12 @@ type Deps struct { // The configuration to use Cfg config.Provider `json:"-"` - // The file cache to use. + // The file caches to use. FileCaches filecache.Caches + // The memory cache to use. + MemCache *memcache.Cache + // The translation func to use Translate func(translationID string, args ...interface{}) string `json:"-"` @@ -158,6 +163,13 @@ type ResourceProvider interface { Clone(deps *Deps) error } +// Stop stops all running caches etc. +func (d *Deps) Stop() { + if d.MemCache != nil { + d.MemCache.Stop() + } +} + func (d *Deps) Tmpl() tpl.TemplateHandler { return d.tmpl } @@ -236,11 +248,12 @@ func New(cfg DepsCfg) (*Deps, error) { if err != nil { return nil, errors.WithMessage(err, "failed to create file caches from configuration") } + memCache := memcache.New(memcache.Config{Running: cfg.Running}) errorHandler := &globalErrHandler{} buildState := &BuildState{} - resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) + resourceSpec, err := resources.NewSpec(ps, fileCaches, memCache, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -277,6 +290,7 @@ func New(cfg DepsCfg) (*Deps, error) { Language: cfg.Language, Site: cfg.Site, FileCaches: fileCaches, + MemCache: memCache, BuildStartListeners: &Listeners{}, BuildState: buildState, Timeout: time.Duration(timeoutms) * time.Millisecond, @@ -311,7 +325,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er // The resource cache is global so reuse. // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) + d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.MemCache, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index 076d9161ccf..f001e36168b 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -371,9 +371,18 @@ Set `titleCaseStyle` to specify the title style used by the [title](/functions/t ## Configuration Environment Variables +### Control Parallel Processing + HUGO_NUMWORKERMULTIPLIER : Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used. +### Control Memory Usage + +HUGO_MEMORYLIMIT {{< new-in "0.75.0" >}} +: A value in Gigabytes (e.g. "1.5") to control the upper memory limit for Hugo's memory use. If not set, it will set aside a quarter of the total system memory. +Note that this does not represent a hard promise that Hugo will not use more memory than configured, but we will try our best. You may need to tune this setting depending on your site and/or system. + + ## Configuration Lookup Order Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior: diff --git a/go.mod b/go.mod index 161333daeca..88e1c771559 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93 github.com/gorilla/websocket v1.4.1 github.com/jdkato/prose v1.1.1 + github.com/karlseguin/ccache/v2 v2.0.7-0.20200814031513-0dbf3f125f13 github.com/kr/pretty v0.2.0 // indirect github.com/kyokomi/emoji v2.2.1+incompatible github.com/magefile/mage v1.9.0 @@ -39,6 +40,7 @@ require ( github.com/nicksnyder/go-i18n v1.10.0 github.com/niklasfasching/go-org v1.3.1 github.com/olekukonko/tablewriter v0.0.4 + github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 github.com/pelletier/go-toml v1.6.0 // indirect github.com/pkg/errors v0.9.1 github.com/rogpeppe/go-internal v1.5.1 diff --git a/go.sum b/go.sum index e47c224795f..85a555d97ff 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,22 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karlseguin/ccache v1.0.1 h1:0gpC6z1qtv0cKmsi5Su5tTB6bJ2vm9bfOLACpDEB/Ro= +github.com/karlseguin/ccache v1.0.2-0.20200216035407-d9aec58960c7 h1:JLz65tG+cbiJ/Yj46FF3rmVmH61g6TI8YWq8p0AX5Zc= +github.com/karlseguin/ccache v1.0.2-0.20200216035407-d9aec58960c7/go.mod h1:bm6z3svDxOYoWqVvk2JmnwOr6dtrTru4/MmlXksuQxk= +github.com/karlseguin/ccache v1.0.2-0.20200626122230-40275a30c888 h1:Pgq9C5Cc/VUYV3Jm9JhzWNPzj5ghtCZ+0CgvtVRx4w4= +github.com/karlseguin/ccache v1.0.2-0.20200626122230-40275a30c888/go.mod h1:bm6z3svDxOYoWqVvk2JmnwOr6dtrTru4/MmlXksuQxk= +github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= +github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= +github.com/karlseguin/ccache/v2 v2.0.6 h1:jFCLz4bF4EPfuCcvESAgYNClkEb31LV3WzyOwLlFz7w= +github.com/karlseguin/ccache/v2 v2.0.6/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200812003947-223703f7f0b5 h1:6eBqXgUzzakiY3AykWKzsqkf/qbaNJdOG5I31JxgiX0= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200812003947-223703f7f0b5/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200813023720-f63031fa4034 h1:rdkclwEmWUb5u/1hcFbrPEe+rDhILqJudpYm2HsCcs0= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200813023720-f63031fa4034/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200814031513-0dbf3f125f13 h1:IRtbBdf9nezZrJHWPJ1atlyY7hByAnLd/Vp0zrr3LV0= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200814031513-0dbf3f125f13/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -285,6 +301,8 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 h1:MfIUBZ1bz7TgvQLVa/yPJZOGeKEgs6eTKUjz3zB4B+U= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= @@ -388,6 +406,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/helpers/general.go b/helpers/general.go index 80e303087a5..377cba65578 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -61,6 +61,24 @@ func FindAvailablePort() (*net.TCPAddr, error) { return nil, err } +// FormatByteCount pretty formats b. +func FormatByteCount(bc uint64) string { + const ( + Gigabyte = 1 << 30 + Megabyte = 1 << 20 + Kilobyte = 1 << 10 + ) + switch { + case bc > Gigabyte || -bc > Gigabyte: + return fmt.Sprintf("%.2f GB", float64(bc)/Gigabyte) + case bc > Megabyte || -bc > Megabyte: + return fmt.Sprintf("%.2f MB", float64(bc)/Megabyte) + case bc > Kilobyte || -bc > Kilobyte: + return fmt.Sprintf("%.2f KB", float64(bc)/Kilobyte) + } + return fmt.Sprintf("%d B", bc) +} + // InStringArray checks if a string is an element of a slice of strings // and returns a boolean value. func InStringArray(arr []string, el string) bool { diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 79d89a88b56..9bd44185712 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -40,6 +40,7 @@ const ( metaKeyBaseDir = "baseDir" // Abs base directory of source file. metaKeyMountRoot = "mountRoot" metaKeyModule = "module" + metaKeyComponent = "component" metaKeyOriginalFilename = "originalFilename" metaKeyName = "name" metaKeyPath = "path" @@ -132,6 +133,10 @@ func (f FileMeta) MountRoot() string { return f.stringV(metaKeyMountRoot) } +func (f FileMeta) Component() string { + return f.stringV(metaKeyComponent) +} + func (f FileMeta) Module() string { return f.stringV(metaKeyModule) } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index ea3ef003ecc..a031f1ec04d 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -41,8 +41,8 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { for _, rm := range rms { (&rm).clean() - fromBase := files.ResolveComponentFolder(rm.From) - if fromBase == "" { + component := files.ResolveComponentFolder(rm.From) + if component == "" { panic("unrecognised component folder in" + rm.From) } @@ -58,7 +58,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { return nil, err } // Extract "blog" from "content/blog" - rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) + rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, component), filepathSeparator) if rm.Meta == nil { rm.Meta = make(FileMeta) } @@ -66,6 +66,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rm.Meta[metaKeyBaseDir] = rm.ToBasedir rm.Meta[metaKeyMountRoot] = rm.path rm.Meta[metaKeyModule] = rm.Module + rm.Meta[metaKeyComponent] = component meta := copyFileMeta(rm.Meta) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index b32f808c971..a7af2a290c0 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -251,6 +251,13 @@ func (m *pageMap) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resour } target := strings.TrimPrefix(meta.Path(), owner.File().Dir()) + tbase := meta.TranslationBaseNameWithExt() + if tbase != "" { + dir, base := filepath.Split(target) + if base != tbase { + target = filepath.Join(dir, tbase) + } + } return owner.s.ResourceSpec.New( resources.ResourceSourceDescriptor{ diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 57a95a03713..8a735f49f51 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -20,6 +20,7 @@ import ( "os" "path" "path/filepath" + "sort" "strings" "sync" @@ -238,6 +239,19 @@ func (s SourceFilesystems) IsAsset(filename string) bool { return s.Assets.Contains(filename) } +// CollectResourcePaths collects paths relative to their component root from +// the data, content and assets filesystems. +func (s SourceFilesystems) CollectResourcePaths(filename string) []string { + var paths []string + + for _, fs := range []*SourceFilesystem{s.Assets, s.Content, s.Data} { + paths = append(paths, fs.collectRelativePaths(filename)...) + } + + return uniqueStringsSorted(paths) + +} + // IsI18n returns true if the given filename is a member of the i18n filesystem. func (s SourceFilesystems) IsI18n(filename string) bool { return s.I18n.Contains(filename) @@ -257,21 +271,40 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { // MakePathRelative creates a relative path from the given filename. // It will return an empty string if the filename is not a member of this filesystem. +// TODO(bep) there may be multiple, see https://github.com/gohugoio/hugo/issues/7563 +// See collectRelativePaths func (d *SourceFilesystem) MakePathRelative(filename string) string { + paths := d.collectRelativePaths(filename) + if paths == nil { + return "" + } + return paths[0] + +} +func (d *SourceFilesystem) collectRelativePaths(filename string) []string { + var paths []string for _, dir := range d.Dirs { meta := dir.(hugofs.FileMetaInfo).Meta() currentPath := meta.Filename() - - if strings.HasPrefix(filename, currentPath) { - rel := strings.TrimPrefix(filename, currentPath) + if rel := relFilename(currentPath, filename); rel != "" { if mp := meta.Path(); mp != "" { rel = filepath.Join(mp, rel) } - return strings.TrimPrefix(rel, filePathSeparator) + paths = append(paths, rel) } } - return "" + return paths +} + +func relFilename(dirname, filename string) string { + if !strings.HasSuffix(dirname, filePathSeparator) { + dirname += filePathSeparator + } + if !strings.HasPrefix(filename, dirname) { + return "" + } + return strings.TrimPrefix(filename, dirname) } func (d *SourceFilesystem) RealFilename(rel string) string { @@ -743,3 +776,21 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollect return b.createOverlayFs(collector, mounts[1:]) } + +func uniqueStringsSorted(s []string) []string { + if len(s) == 0 { + return nil + } + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 67ee10e0978..b1d94c4eed4 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -51,6 +51,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Make sure we don't trigger rebuilds in parallel. h.runningMu.Lock() defer h.runningMu.Unlock() + } else { + defer func() { + h.Stop() + }() } ctx, task := trace.NewTask(context.Background(), "Build") diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 8d0872bd5e0..88ac32fa10b 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -2,6 +2,7 @@ package hugolib import ( "fmt" + "os" "strings" "testing" @@ -11,6 +12,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource_transformers/tocss/scss" "github.com/fortytw2/leaktest" "github.com/fsnotify/fsnotify" @@ -417,15 +419,18 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { func TestMultiSitesRebuild(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 - // This leaktest seems to be a little bit shaky on Travis. - if !isCI() { - defer leaktest.CheckTimeout(t, 10*time.Second)() - } c := qt.New(t) b := newMultiSiteTestDefaultBuilder(t).Running().CreateSites().Build(BuildCfg{}) + defer b.H.Stop() + + // This leaktest seems to be a little bit shaky on Travis. + if !isCI() { + defer leaktest.CheckTimeout(t, 10*time.Second)() + } + sites := b.H.Sites fs := b.Fs @@ -1478,3 +1483,71 @@ func TestRebuildOnAssetChange(t *testing.T) { b.Build(BuildCfg{}) b.AssertFileContent("public/index.html", `changed data`) } + +func TestCacheOnRebuild(t *testing.T) { + c := qt.New(t) + + createSitesBuilder := func(t testing.TB, prepare func(b *sitesBuilder)) *sitesBuilder { + b := newTestSitesBuilder(t).Running() + prepare(b) + b.Build(BuildCfg{}) + return b + + } + + c.Run("transform.Unmarshal with temporary Resource", func(c *qt.C) { + templ := ` + {{ $m := "a: 1" | resources.FromString "a.yaml" | transform.Unmarshal }} + NM: {{ $m }} + + ` + b := createSitesBuilder(c, func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", templ) + }) + + b.AssertFileContent("public/index.html", `NM: map[a:1]`) + + templ = strings.ReplaceAll(templ, "a: 1", "a: 2") + templ = strings.ReplaceAll(templ, "NM:", "NM2:") + b.EditFiles("layouts/index.html", templ) + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `NM2: map[a:2]`) + + }) + + c.Run("SCSS with multiple roots", func(c *qt.C) { + if !scss.Supports() { + c.Skip("Skip SCSS") + } + + b, workDir, clean := newTestSitesBuilderWithOSFs(c, "scss-multiroot") + defer clean() + b.Running() + + assetsDir := filepath.Join(workDir, "assets") + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root1"), 0777), qt.IsNil) + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root2"), 0777), qt.IsNil) + + b.WithTemplatesAdded("index.html", ` +{{ $css := resources.Get "root2/main.scss" | toCSS }} +CSS: {{ $css.Content | safeHTML }} +`) + + b.WithSourceFile(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: blue;`) + b.WithSourceFile(filepath.Join(assetsDir, "root2/main.scss"), `@import "../root1/_vars"; + +body { color: $base-color; } +`) + + b.Build(BuildCfg{}) + b.AssertFileContent("public/index.html", `color: blue;`) + + b.EditFiles(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: red;`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `color: red;`) + }) + +} diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 4fe4960422c..80885b5d67c 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -89,7 +89,7 @@ languageName = "Nynorsk" s2h := s2.getPage(page.KindHome) c.Assert(s2h.Permalink(), qt.Equals, "https://example.fr/") - b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /docs/text/pipes.txt") + b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /text/pipes.txt") b.AssertFileContent("public/fr/text/pipes.txt", "Hugo Pipes") b.AssertFileContent("public/en/index.html", "Default Home Page", "String Resource: /docs/text/pipes.txt") b.AssertFileContent("public/en/text/pipes.txt", "Hugo Pipes") diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index fa420a025aa..5af55e738f0 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -347,7 +347,7 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { b.AssertFileContent("public/en/bc/data1.json", "data1") b.AssertFileContent("public/en/bc/data2.json", "data2") b.AssertFileContent("public/en/bc/logo-bc.png", "logo") - b.AssertFileContent("public/nn/bc/data1.nn.json", "data1.nn") + b.AssertFileContent("public/nn/bc/data1.json", "data1.nn") b.AssertFileContent("public/nn/bc/data2.json", "data2") b.AssertFileContent("public/nn/bc/logo-bc.png", "logo") @@ -1320,7 +1320,7 @@ bundle min min key: {{ $jsonMinMin.Key }} b.AssertFileContent(index, fmt.Sprintf("bundle min min min: /bundle%d/data.min.min.min.json", i), - fmt.Sprintf("bundle min min key: /bundle%d/data.min.min.json", i), + fmt.Sprintf("bundle min min key: bundle%d/content/en/data_c9a5f8c93697691d6f8a298931d285f7.json", i), ) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.min.min.json", i)), qt.Equals, true) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.json", i)), qt.Equals, false) diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 020e243c505..f5183f2260b 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -178,10 +178,6 @@ func (c *pagesCollector) Collect() (collectErr error) { } for dir := range dirs { - for _, pm := range c.contentMap.pmaps { - pm.s.ResourceSpec.DeleteBySubstring(dir.dirname) - } - switch dir.tp { case bundleLeaf: collectErr = c.collectDir(dir.dirname, true, nil) diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 97d4f17bab7..066c788fd16 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -53,6 +53,8 @@ type Paths struct { PublishDir string + IsMultiHost bool + // When in multihost mode, this returns a list of base paths below PublishDir // for each language. MultihostTargetBasePaths []string @@ -142,7 +144,8 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { } var multihostTargetBasePaths []string - if languages.IsMultihost() { + isMultiHost := languages.IsMultihost() + if isMultiHost { for _, l := range languages { multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang) } @@ -171,6 +174,7 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { Language: language, Languages: languages, LanguagesDefaultFirst: languagesDefaultFirst, + IsMultiHost: isMultiHost, MultihostTargetBasePaths: multihostTargetBasePaths, PaginatePath: cfg.GetString("paginatePath"), diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index c687ca3421b..e208d4d2ed2 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -350,7 +350,7 @@ Edited content. `) b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil) - b.H.ResourceSpec.ClearCaches() + b.H.MemCache.Clear() } } diff --git a/hugolib/resource_change_test.go b/hugolib/resource_change_test.go new file mode 100644 index 00000000000..941df8e7ea0 --- /dev/null +++ b/hugolib/resource_change_test.go @@ -0,0 +1,322 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "strings" + "testing" +) + +func TestResourceEditMetadata(t *testing.T) { + b := newTestSitesBuilder(t).Running() + + content := `+++ +title = "My Bundle With TOML Meta" + +[[resources]] +src = "**.toml" +title = "My TOML :counter" ++++ + +Content. +` + + b.WithContent( + "bundle/index.md", content, + "bundle/my1.toml", `a = 1`, + "bundle/my2.toml", `a = 2`) + + b.WithTemplatesAdded("index.html", ` +{{ $bundle := site.GetPage "bundle" }} +{{ $toml := $bundle.Resources.GetMatch "*.toml" }} +TOML: {{ $toml.Title }} + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My TOML 1") + + b.EditFiles("content/bundle/index.md", strings.ReplaceAll(content, "My TOML", "My Changed TOML 1")) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My Changed TOML") + +} + +func TestResourceCacheSimpleTest(t *testing.T) { + conf := ` +baseURL = "https://example.org" + +defaultContentLanguage = "en" + +[module] +[[module.mounts]] +source = "content/cen" +target = "content" +lang="en" +[[module.mounts]] +source = "content/cno" +target = "content" +lang="no" +[[module.mounts]] +source = "assets" +target = "assets" +[[module.mounts]] +source = "assets_common" +target = "assets/aen" +[[module.mounts]] +source = "assets_common" +target = "assets/ano" + +[languages] +[languages.en] +weight = 1 + +[languages.no] +weight = 2 + +` + b := newTestSitesBuilder(t).WithConfigFile("toml", conf).Running() + + b.WithSourceFile( + "content/cen/bundle/index.md", "---\ntitle: En Bundle\n---", + "content/cen/bundle/data1.json", `{ "data1": "en" }`, + "content/cen/bundle/data2.json", `{ "data2": "en" }`, + "content/cno/bundle/index.md", "---\ntitle: No Bundle\n---", + "content/cno/bundle/data1.json", `{ "data1": "no" }`, + "content/cno/bundle/data3.json", `{ "data3": "no" }`, + ) + + b.WithSourceFile("assets_common/data/common.json", `{ + "Hugo": "Rocks!", + }`) + + b.WithSourceFile("assets/data/mydata.json", `{ + "a": 32, + }`) + + b.WithTemplatesAdded("index.html", ` +{{ $data := resources.Get "data/mydata.json" }} +{{ template "print-resource" ( dict "title" "data" "r" $data ) }} +{{ $dataMinified := $data | minify }} +{{ template "print-resource" ( dict "title" "data-minified" "r" $dataMinified ) }} +{{ $dataUnmarshaled := $dataMinified | transform.Unmarshal }} +Data Unmarshaled: {{ $dataUnmarshaled }} +{{ $bundle := site.GetPage "bundle" }} +{{ range (seq 3) }} +{{ $i := . }} +{{ with $bundle.Resources.GetMatch (printf "data%d.json" . ) }} +{{ $minified := . | minify }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d" $i) "r" . ) }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d min" $i) "r" $minified ) }} +{{ end }} +{{ end }} +{{ $common1 := resources.Get "aen/data/common.json" }} +{{ $common2 := resources.Get "ano/data/common.json" }} +{{ template "print-resource" ( dict "title" "common1" "r" $common1 ) }} +{{ template "print-resource" ( dict "title" "common2" "r" $common2 ) }} +{{ define "print-resource" }}{{ .title }}|{{ .r.RelPermalink }}|{{ .r.Key }}|{{ .r.Content | safeHTML }}|{{ end }} + + + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +Data Unmarshaled: map[a:32] +bundle data 1|/bundle/data1.json|bundle/content/en/data1.json|{ "data1": "en" }| +bundle data 1 min|/bundle/data1.min.json|bundle/content/en/data1_d3f53f09220d597dac26fe7840c31fc9.json|{"data1":"en"}| +bundle data 3|/bundle/data3.json|bundle/content/en/data3.json|{ "data3": "no" }| +bundle data 3 min|/bundle/data3.min.json|bundle/content/en/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json| +common2|/ano/data/common.json|ano/data/common.json| +`) + + b.AssertFileContent("public/no/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +bundle data 1|/no/bundle/data1.json|bundle/content/no/data1.json|{ "data1": "no" }| +bundle data 2|/no/bundle/data2.json|bundle/content/no/data2.json|{ "data2": "en" }| + bundle data 3|/no/bundle/data3.json|bundle/content/no/data3.json|{ "data3": "no" }| +bundle data 3 min|/no/bundle/data3.min.json|bundle/content/no/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json +`) + + b.EditFiles("assets/data/mydata.json", `{ "a": 42 }`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data|/data/mydata.json|data/mydata.json|{ "a": 42 }| +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":42}| +Data Unmarshaled: map[a:42] +`) + +} + +func TestResourceCacheMultihost(t *testing.T) { + toLang := func(format, lang string) string { + return fmt.Sprintf(format, lang) + } + + addContent := func(b *sitesBuilder, contentPath func(path, lang string) string) { + b.WithNoContentAdded() + for _, lang := range []string{"en", "fr"} { + b.WithSourceFile( + contentPath("b1/index.md", lang), toLang("---\ntitle: Bundle 1 %s\n---", lang), + contentPath("b1/styles/style11.css", lang), toLang(".%s1: { color: blue };", lang), + contentPath("b1/styles/style12.css", lang), toLang(".%s2: { color: red };", lang), + ) + b.WithSourceFile( + contentPath("b2/index.md", lang), toLang("---\ntitle: Bundle 2 %s\n---", lang), + contentPath("b2/styles/style21.css", lang), toLang(".%s21: { color: green };", lang), + contentPath("b2/styles/style22.css", lang), toLang(".%s22: { color: orange };", lang), + ) + } + } + + addTemplates := func(b *sitesBuilder) { + b.WithTemplates("_default/single.html", ` +{{ template "print-page" (dict "page" . "title" "Self") }} +{{ $other := site.Sites.First.GetPage "b1" }} +{{ template "print-page" (dict "page" $other "title" "Other") }} + + +{{ define "print-page" }} +{{ $p := .page }} +{{ $title := .title }} +{{ $styles := $p.Resources.Match "**/style1*.css" }} +{{ if $styles }} +{{ $firststyle := index $styles 0 }} +{{ $mystyles := $styles | resources.Concat "mystyles.css" }} +{{ $title }} Mystyles First CSS: {{ $firststyle.RelPermalink }}|Key: {{ $firststyle.Key }}|{{ $firststyle.Content }}| +{{ $title }} Mystyles CSS: {{ $mystyles.RelPermalink }}|Key: {{ $mystyles.Key }}|{{ $mystyles.Content }}| +{{ end }} +{{ $title }} Bundle: {{ $p.Permalink }} +{{ $style := $p.Resources.GetMatch "**.css" }} +{{ $title }} CSS: {{ $style.RelPermalink }}|Key: {{ $style.Key }}|{{ $style.Content }}| +{{ $minified := $style | minify }} +{{ $title }} Minified CSS: {{ $minified.RelPermalink }}|Key: {{ $minified.Key }}|{{ $minified.Content }} +{{ end }} + +`) + } + + assertContent := func(b *sitesBuilder) { + + otherAssert := `Other Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Other Bundle: https://example.com/b1/ +Other CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue }; +Other Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue}` + + b.AssertFileContent("public/fr/b1/index.html", ` +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red }; +Self Bundle: https://example.fr/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/fr/styles/style11.css|.fr1: { color: blue }; +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/fr/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.fr1:{color:blue} +`, + otherAssert) + b.AssertFileContent("public/en/b1/index.html", ` +Self Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red };| +Self Bundle: https://example.com/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue} +`, otherAssert) + + b.AssertFileContent("public/fr/b2/index.html", ` +Self Bundle: https://example.fr/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/fr/styles/style21.css|.fr21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/fr/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.fr21:{color:green} +`, otherAssert) + + b.AssertFileContent("public/en/b2/index.html", ` +Self Bundle: https://example.com/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/en/styles/style21.css|.en21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/en/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.en21:{color:green} +`, otherAssert) + + } + + t.Run("Default content", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false +contentDir = "content" + +[Languages] +[Languages.en] +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + fmt.Println(b.workingDir) + addContent(b, func(path, lang string) string { + path = strings.Replace(path, ".", "."+lang+".", 1) + path = "content/" + path + return path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + + t.Run("Content dir per language", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false + +[Languages] +[Languages.en] +contentDir = "content_en" +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +contentDir = "content_fr" +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + addContent(b, func(path, lang string) string { + return "content_" + lang + "/" + path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + +} diff --git a/hugolib/site.go b/hugolib/site.go index ac65931d0c8..fe6194c93c6 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -28,12 +28,12 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/constants" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/converter/hooks" @@ -1027,11 +1027,12 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro logger = helpers.NewDistinctFeedbackLogger() ) - var cachePartitions []string + // Paths relative to their component folder in data, assets or content. + var resourcePaths []string for _, ev := range events { - if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { - cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) + if paths := s.BaseFs.CollectResourcePaths(ev.Name); paths != nil { + resourcePaths = append(resourcePaths, paths...) } id, found := s.eventToIdentity(ev) @@ -1075,10 +1076,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro return err } - // These in memory resource caches will be rebuilt on demand. - for _, s := range s.h.Sites { - s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) - } + h.MemCache.ClearOn(memcache.ClearOnRebuild, resourcePaths...) if tmplChanged || i18nChanged { sites := s.h.Sites diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 2af4691d173..fb17738ad7d 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -92,7 +92,8 @@ type sitesBuilder struct { // Consider this in relation to using the BaseFs.PublishFs to all publishing. workingDir string - addNothing bool + addNothing bool + addNoContent bool // Base data/content contentFilePairs []filenameContent templateFilePairs []filenameContent @@ -126,6 +127,21 @@ func newTestSitesBuilder(t testing.TB) *sitesBuilder { dumper: litterOptions, rnd: rand.New(rand.NewSource(time.Now().Unix()))} } +func newTestSitesBuilderWithOSFs(t testing.TB, testname string) (*sitesBuilder, string, func()) { + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, testname) + if err != nil { + t.Fatal(err) + } + v := viper.New() + v.Set("workingDir", workDir) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) + + return b, workDir, clean +} + func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder { c := qt.New(t) @@ -154,6 +170,11 @@ func (s *sitesBuilder) WithNothingAdded() *sitesBuilder { return s } +func (s *sitesBuilder) WithNoContentAdded() *sitesBuilder { + s.addNoContent = true + return s +} + func (s *sitesBuilder) WithLogger(logger *loggers.Logger) *sitesBuilder { s.logger = logger return s @@ -502,6 +523,7 @@ func (s *sitesBuilder) LoadConfig() error { } func (s *sitesBuilder) CreateSitesE() error { + if !s.addNothing { if _, ok := s.Fs.Source.(*afero.OsFs); ok { for _, dir := range []string{ @@ -529,7 +551,6 @@ func (s *sitesBuilder) CreateSitesE() error { s.writeFilePairs("data", s.dataFilePairs) s.writeFilePairs("content", s.contentFilePairs) s.writeFilePairs("layouts", s.templateFilePairs) - } if err := s.LoadConfig(); err != nil { @@ -677,7 +698,7 @@ hello: } ) - if len(s.contentFilePairs) == 0 { + if len(s.contentFilePairs) == 0 && !s.addNoContent { s.writeFilePairs("content", s.createFilenameContent(defaultContent)) } diff --git a/modules/client.go b/modules/client.go index 8ee9f588aaf..a13e7688105 100644 --- a/modules/client.go +++ b/modules/client.go @@ -342,7 +342,7 @@ var verifyErrorDirRe = regexp.MustCompile(`dir has been modified \((.*?)\)`) // which are stored in a local downloaded source cache, have not been // modified since being downloaded. func (c *Client) Verify(clean bool) error { - // TODO1 add path to mod clean + // TODO(bep) add path to mod clean err := c.runVerify() if err != nil { diff --git a/resources/image.go b/resources/image.go index d1d9f650d67..5b3905de67d 100644 --- a/resources/image.go +++ b/resources/image.go @@ -367,7 +367,7 @@ func (i *imageResource) getImageMetaCacheTargetPath() string { df.dir = filepath.Dir(fi.Meta().Path()) } p1, _ := helpers.FileAndExt(df.file) - h, _ := i.hash() + h := i.hash() idStr := helpers.HashString(h, i.size(), imageMetaVersionNumber, cfg) return path.Join(df.dir, fmt.Sprintf("%s_%s.json", p1, idStr)) } @@ -378,7 +378,7 @@ func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile p2 = conf.TargetFormat.DefaultExtension() } - h, _ := i.hash() + h := i.hash() idStr := fmt.Sprintf("_hu%s_%d", h, i.size()) // Do not change for no good reason. diff --git a/resources/image_cache.go b/resources/image_cache.go index 1888b457f59..b1db71cbea6 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -17,8 +17,8 @@ import ( "image" "io" "path/filepath" - "strings" - "sync" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/images" @@ -30,36 +30,7 @@ type imageCache struct { pathSpec *helpers.PathSpec fileCache *filecache.Cache - - mu sync.RWMutex - store map[string]*resourceAdapter -} - -func (c *imageCache) deleteIfContains(s string) { - c.mu.Lock() - defer c.mu.Unlock() - s = c.normalizeKeyBase(s) - for k := range c.store { - if strings.Contains(k, s) { - delete(c.store, k) - } - } -} - -// The cache key is a lowecase path with Unix style slashes and it always starts with -// a leading slash. -func (c *imageCache) normalizeKey(key string) string { - return "/" + c.normalizeKeyBase(key) -} - -func (c *imageCache) normalizeKeyBase(key string) string { - return strings.Trim(strings.ToLower(filepath.ToSlash(key)), "/") -} - -func (c *imageCache) clear() { - c.mu.Lock() - defer c.mu.Unlock() - c.store = make(map[string]*resourceAdapter) + mCache memcache.Getter } func (c *imageCache) getOrCreate( @@ -67,101 +38,92 @@ func (c *imageCache) getOrCreate( createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) { relTarget := parent.relTargetPathFromConfig(conf) memKey := parent.relTargetPathForRel(relTarget.path(), false, false, false) - memKey = c.normalizeKey(memKey) + memKey = memcache.CleanKey(memKey) - // For the file cache we want to generate and store it once if possible. - fileKeyPath := relTarget - if fi := parent.root.getFileInfo(); fi != nil { - fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) - } - fileKey := fileKeyPath.path() + v, err := c.mCache.GetOrCreate(memKey, func() memcache.Entry { + // For the file cache we want to generate and store it once if possible. + fileKeyPath := relTarget + if fi := parent.root.getFileInfo(); fi != nil { + fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) + } + fileKey := fileKeyPath.path() - // First check the in-memory store, then the disk. - c.mu.RLock() - cachedImage, found := c.store[memKey] - c.mu.RUnlock() + var img *imageResource - if found { - return cachedImage, nil - } + // These funcs are protected by a named lock. + // read clones the parent to its new name and copies + // the content to the destinations. + read := func(info filecache.ItemInfo, r io.ReadSeeker) error { + img = parent.clone(nil) + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - var img *imageResource + if err := img.InitConfig(r); err != nil { + return err + } - // These funcs are protected by a named lock. - // read clones the parent to its new name and copies - // the content to the destinations. - read := func(info filecache.ItemInfo, r io.ReadSeeker) error { - img = parent.clone(nil) - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) + r.Seek(0, 0) - if err := img.InitConfig(r); err != nil { - return err - } + w, err := img.openDestinationsForWriting() + if err != nil { + return err + } - r.Seek(0, 0) + if w == nil { + // Nothing to write. + return nil + } + + defer w.Close() + _, err = io.Copy(w, r) - w, err := img.openDestinationsForWriting() - if err != nil { return err } - if w == nil { - // Nothing to write. - return nil - } + // create creates the image and encodes it to the cache (w). + create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + defer w.Close() - defer w.Close() - _, err = io.Copy(w, r) + var conv image.Image + img, conv, err = createImage() + if err != nil { + return + } + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - return err - } + return img.EncodeTo(conf, conv, w) + } + + // Now look in the file cache. - // create creates the image and encodes it to the cache (w). - create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { - defer w.Close() + // The definition of this counter is not that we have processed that amount + // (e.g. resized etc.), it can be fetched from file cache, + // but the count of processed image variations for this site. + c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - var conv image.Image - img, conv, err = createImage() + _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { - return + return memcache.Entry{Err: err} } - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) - return img.EncodeTo(conf, conv, w) - } + // The file is now stored in this cache. + img.setSourceFs(c.fileCache.Fs) - // Now look in the file cache. + imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - // The definition of this counter is not that we have processed that amount - // (e.g. resized etc.), it can be fetched from file cache, - // but the count of processed image variations for this site. - c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) + return memcache.Entry{Value: imgAdapter} + }) - _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { return nil, err } + return v.(*resourceAdapter), nil - // The file is now stored in this cache. - img.setSourceFs(c.fileCache.Fs) - - c.mu.Lock() - if cachedImage, found = c.store[memKey]; found { - c.mu.Unlock() - return cachedImage, nil - } - - imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - c.store[memKey] = imgAdapter - c.mu.Unlock() - - return imgAdapter, nil } -func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { - return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)} +func newImageCache(fileCache *filecache.Cache, memCache *memcache.Cache, ps *helpers.PathSpec) *imageCache { + return &imageCache{fileCache: fileCache, mCache: memCache.GetOrCreatePartition("images", memcache.ClearOnChange), pathSpec: ps} } diff --git a/resources/image_test.go b/resources/image_test.go index f98d9f4bb24..2826ea98405 100644 --- a/resources/image_test.go +++ b/resources/image_test.go @@ -352,7 +352,7 @@ func TestImageResizeInSubPath(t *testing.T) { c.Assert(spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) // Cleare mem cache to simulate reading from the file cache. - spec.imageCache.clear() + spec.imageCache.mCache.Clear() resizedAgain, err := image.Resize("101x101") c.Assert(err, qt.IsNil) diff --git a/resources/resource.go b/resources/resource.go index acdf2d744ec..5209c0a9796 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -22,6 +22,8 @@ import ( "path/filepath" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/resources/internal" "github.com/gohugoio/hugo/common/herrors" @@ -172,8 +174,6 @@ func (commonResource) Slice(in interface{}) (interface{}, error) { return nil, fmt.Errorf("type %T is not a Resource", v) } groups[i] = g - { - } } return groups, nil default: @@ -197,8 +197,24 @@ type fileInfo interface { setSourceFilename(string) setSourceFs(afero.Fs) getFileInfo() hugofs.FileMetaInfo - hash() (string, error) size() int + hashProvider +} + +type hashProvider interface { + hash() string +} + +type resourceOrigin struct { + stale bool +} + +func (s *resourceOrigin) MarkStale() { + s.stale = true +} + +func (s *resourceOrigin) IsStale() bool { + return s.stale } // genericResource represents a generic linkable resource. @@ -235,7 +251,25 @@ func (l *genericResource) Data() interface{} { } func (l *genericResource) Key() string { - return l.RelPermalink() + // TODO1 consider repating the section in the path segment. + + if l.fi != nil { + // Create a key that at least shares the base folder with the source, + // to facilitate effective cache busting on changes. + meta := l.fi.Meta() + p := meta.Path() + if p != "" { + d, _ := filepath.Split(p) + p = path.Join(d, l.relTargetDirFile.file) + key := memcache.CleanKey(p) + key = memcache.InsertKeyPathElements(key, meta.Component(), meta.Lang()) + + return key + } + } + key := memcache.CleanKey(l.RelPermalink()) + + return key } func (l *genericResource) MediaType() media.Type { @@ -417,6 +451,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour } fpath, fname := path.Split(u.targetPath) + r.resourcePathDescriptor.relTargetDirFile = dirFile{dir: fpath, file: fname} r.mergeData(u.data) @@ -623,26 +658,25 @@ func (fi *resourceFileInfo) setSourceFs(fs afero.Fs) { fi.sourceFs = fs } -func (fi *resourceFileInfo) hash() (string, error) { +func (fi *resourceFileInfo) hash() string { var err error fi.h.init.Do(func() { var hash string var f hugio.ReadSeekCloser f, err = fi.ReadSeekCloser() if err != nil { - err = errors.Wrap(err, "failed to open source file") - return + panic(fmt.Sprintf("failed to open source file: %s", err)) } defer f.Close() hash, err = helpers.MD5FromFileFast(f) if err != nil { - return + panic(err) } fi.h.value = hash }) - return fi.h.value, err + return fi.h.value } func (fi *resourceFileInfo) size() int { diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 13ffc5ae3ea..9f4dfdedf29 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -179,6 +179,36 @@ type UnmarshableResource interface { Identifier } +// Staler controls stale state of a Resource. A stale resource should be discarded. +type Staler interface { + MarkStale() + StaleInfo +} + +// StaleInfo tells if a resource is marked as stale. +type StaleInfo interface { + IsStale() bool +} + +// IsStaleAny reports whether any of the os is marked as stale. +func IsStaleAny(os ...interface{}) bool { + for _, o := range os { + if s, ok := o.(StaleInfo); ok && s.IsStale() { + return true + } + } + return false +} + +// MarkStale will mark any of the oses as stale, if possible. +func MarkStale(os ...interface{}) { + for _, o := range os { + if s, ok := o.(Staler); ok { + s.MarkStale() + } + } +} + type resourceTypesHolder struct { mediaType media.Type resourceType string diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 47822a7f506..e36d0435471 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -16,25 +16,13 @@ package resources import ( "encoding/json" "io" - "path" - "path/filepath" - "strings" "sync" - "github.com/gohugoio/hugo/helpers" - - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/cache/filecache" - - "github.com/BurntSushi/locker" -) - -const ( - CACHE_CLEAR_ALL = "clear_all" - CACHE_OTHER = "other" ) type ResourceCache struct { @@ -42,123 +30,30 @@ type ResourceCache struct { sync.RWMutex - // Either resource.Resource or resource.Resources. - cache map[string]interface{} + // Memory cache with either + // resource.Resource or resource.Resources. + cache memcache.Getter fileCache *filecache.Cache - - // Provides named resource locks. - nlocker *locker.Locker -} - -// ResourceCacheKey converts the filename into the format used in the resource -// cache. -func ResourceCacheKey(filename string) string { - filename = filepath.ToSlash(filename) - return path.Join(resourceKeyPartition(filename), filename) -} - -func resourceKeyPartition(filename string) string { - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") - if ext == "" { - ext = CACHE_OTHER - } - return ext -} - -// Commonly used aliases and directory names used for some types. -var extAliasKeywords = map[string][]string{ - "sass": []string{"scss"}, - "scss": []string{"sass"}, -} - -// ResourceKeyPartitions resolves a ordered slice of partitions that is -// used to do resource cache invalidations. -// -// We use the first directory path element and the extension, so: -// a/b.json => "a", "json" -// b.json => "json" -// -// For some of the extensions we will also map to closely related types, -// e.g. "scss" will also return "sass". -// -func ResourceKeyPartitions(filename string) []string { - var partitions []string - filename = glob.NormalizePath(filename) - dir, name := path.Split(filename) - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(name)), ".") - - if dir != "" { - partitions = append(partitions, strings.Split(dir, "/")[0]) - } - - if ext != "" { - partitions = append(partitions, ext) - } - - if aliases, found := extAliasKeywords[ext]; found { - partitions = append(partitions, aliases...) - } - - if len(partitions) == 0 { - partitions = []string{CACHE_OTHER} - } - - return helpers.UniqueStringsSorted(partitions) -} - -// ResourceKeyContainsAny returns whether the key is a member of any of the -// given partitions. -// -// This is used for resource cache invalidation. -func ResourceKeyContainsAny(key string, partitions []string) bool { - parts := strings.Split(key, "/") - for _, p1 := range partitions { - for _, p2 := range parts { - if p1 == p2 { - return true - } - } - } - return false } -func newResourceCache(rs *Spec) *ResourceCache { +func newResourceCache(rs *Spec, memCache *memcache.Cache) *ResourceCache { return &ResourceCache{ rs: rs, fileCache: rs.FileCaches.AssetsCache(), - cache: make(map[string]interface{}), - nlocker: locker.NewLocker(), + cache: memCache.GetOrCreatePartition("resources", memcache.ClearOnChange), } } -func (c *ResourceCache) clear() { - c.Lock() - defer c.Unlock() - - c.cache = make(map[string]interface{}) - c.nlocker = locker.NewLocker() -} - -func (c *ResourceCache) Contains(key string) bool { - key = c.cleanKey(filepath.ToSlash(key)) - _, found := c.get(key) - return found -} - -func (c *ResourceCache) cleanKey(key string) string { - return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") -} - -func (c *ResourceCache) get(key string) (interface{}, bool) { - c.RLock() - defer c.RUnlock() - r, found := c.cache[key] - return r, found -} - -func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, error)) (resource.Resource, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) +func (c *ResourceCache) GetOrCreate(key string, clearWhen memcache.ClearWhen, f func() (resource.Resource, error)) (resource.Resource, error) { + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + ClearWhen: clearWhen, + } + }) if r == nil || err != nil { return nil, err } @@ -166,43 +61,19 @@ func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, err } func (c *ResourceCache) GetOrCreateResources(key string, f func() (resource.Resources, error)) (resource.Resources, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + } + }) if r == nil || err != nil { return nil, err } return r.(resource.Resources), nil } -func (c *ResourceCache) getOrCreate(key string, f func() (interface{}, error)) (interface{}, error) { - key = c.cleanKey(key) - // First check in-memory cache. - r, found := c.get(key) - if found { - return r, nil - } - // This is a potentially long running operation, so get a named lock. - c.nlocker.Lock(key) - - // Double check in-memory cache. - r, found = c.get(key) - if found { - c.nlocker.Unlock(key) - return r, nil - } - - defer c.nlocker.Unlock(key) - - r, err := f() - if err != nil { - return nil, err - } - - c.set(key, r) - - return r, nil - -} - func (c *ResourceCache) getFilenames(key string) (string, string) { filenameMeta := key + ".json" filenameContent := key + ".content" @@ -255,43 +126,3 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) return fi, fc, err } - -func (c *ResourceCache) set(key string, r interface{}) { - c.Lock() - defer c.Unlock() - c.cache[key] = r -} - -func (c *ResourceCache) DeletePartitions(partitions ...string) { - partitionsSet := map[string]bool{ - // Always clear out the resources not matching any partition. - "other": true, - } - for _, p := range partitions { - partitionsSet[p] = true - } - - if partitionsSet[CACHE_CLEAR_ALL] { - c.clear() - return - } - - c.Lock() - defer c.Unlock() - - for k := range c.cache { - clear := false - for p := range partitionsSet { - if strings.Contains(k, p) { - // There will be some false positive, but that's fine. - clear = true - break - } - } - - if clear { - delete(c.cache, k) - } - } - -} diff --git a/resources/resource_cache_test.go b/resources/resource_cache_test.go deleted file mode 100644 index bcb24102594..00000000000 --- a/resources/resource_cache_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resources - -import ( - "path/filepath" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestResourceKeyPartitions(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - input string - expected []string - }{ - {"a.js", []string{"js"}}, - {"a.scss", []string{"sass", "scss"}}, - {"a.sass", []string{"sass", "scss"}}, - {"d/a.js", []string{"d", "js"}}, - {"js/a.js", []string{"js"}}, - {"D/a.JS", []string{"d", "js"}}, - {"d/a", []string{"d"}}, - {filepath.FromSlash("/d/a.js"), []string{"d", "js"}}, - {filepath.FromSlash("/d/e/a.js"), []string{"d", "js"}}, - } { - c.Assert(ResourceKeyPartitions(test.input), qt.DeepEquals, test.expected, qt.Commentf(test.input)) - } -} - -func TestResourceKeyContainsAny(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - key string - filename string - expected bool - }{ - {"styles/css", "asdf.css", true}, - {"styles/css", "styles/asdf.scss", true}, - {"js/foo.bar", "asdf.css", false}, - } { - c.Assert(ResourceKeyContainsAny(test.key, ResourceKeyPartitions(test.filename)), qt.Equals, test.expected) - } -} diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index 1ea92bea397..818dbaf5294 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -17,9 +17,10 @@ package bundler import ( "fmt" "io" - "path" "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" @@ -81,8 +82,7 @@ func (r *multiReadSeekCloser) Close() error { // Concat concatenates the list of Resource objects. func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resource, error) { - // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(targetPath, memcache.ClearOnRebuild, func() (resource.Resource, error) { var resolvedm media.Type // The given set of resources must be of the same Media Type. diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 4ac20d36e5a..49604ca0720 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -18,7 +18,8 @@ package create import ( "path" "path/filepath" - "strings" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/hugofs/glob" @@ -43,7 +44,7 @@ func New(rs *resources.Spec) *Client { // Get creates a new Resource by opening the given filename in the assets filesystem. func (c *Client) Get(filename string) (resource.Resource, error) { filename = filepath.Clean(filename) - return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(filename), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(filename), memcache.ClearOnChange, func() (resource.Resource, error) { return c.rs.New(resources.ResourceSourceDescriptor{ Fs: c.rs.BaseFs.Assets.Fs, LazyPublish: true, @@ -74,13 +75,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro name = "__match" } - pattern = glob.NormalizePath(pattern) - partitions := glob.FilterGlobParts(strings.Split(pattern, "/")) - if len(partitions) == 0 { - partitions = []string{resources.CACHE_OTHER} - } - key := path.Join(name, path.Join(partitions...)) - key = path.Join(key, pattern) + key := path.Join(name, glob.NormalizePath(pattern)) return c.rs.ResourceCache.GetOrCreateResources(key, func() (resource.Resources, error) { var res resource.Resources @@ -116,7 +111,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + r, err := c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(targetPath), memcache.ClearOnRebuild, func() (resource.Resource, error) { return c.rs.New( resources.ResourceSourceDescriptor{ Fs: c.rs.FileCaches.AssetsCache().Fs, @@ -128,4 +123,9 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro }) + if err == nil { + r.(resource.Staler).MarkStale() + } + return r, err + } diff --git a/resources/resource_metadata_test.go b/resources/resource_metadata_test.go index c79a5002121..0414127b92a 100644 --- a/resources/resource_metadata_test.go +++ b/resources/resource_metadata_test.go @@ -209,12 +209,12 @@ func TestAssignMetadata(t *testing.T) { }}, } { - foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) - logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) - foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) - logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) - foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) - logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) + foo2 = newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) + logo2 = newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) + foo1 = newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) + logo1 = newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) + foo3 = newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) + logo3 = newGenericResource(spec, nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) resources = resource.Resources{ foo2, diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 81eed2f0203..d2abdbe053f 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/config" @@ -46,6 +48,7 @@ import ( func NewSpec( s *helpers.PathSpec, fileCaches filecache.Caches, + memCache *memcache.Cache, incr identity.Incrementer, logger *loggers.Logger, errorHandler herrors.ErrorSender, @@ -89,11 +92,11 @@ func NewSpec( PostProcessResources: make(map[string]postpub.PostPublishedResource), imageCache: newImageCache( fileCaches.ImageCache(), - + memCache, s, )} - rs.ResourceCache = newResourceCache(rs) + rs.ResourceCache = newResourceCache(rs, memCache) return rs, nil @@ -129,57 +132,11 @@ func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { return r.newResourceFor(fd) } -func (r *Spec) CacheStats() string { - r.imageCache.mu.RLock() - defer r.imageCache.mu.RUnlock() - - s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) - - count := 0 - for k := range r.imageCache.store { - if count > 5 { - break - } - s += "\n" + k - count++ - } - - return s -} - -func (r *Spec) ClearCaches() { - r.imageCache.clear() - r.ResourceCache.clear() -} - -func (r *Spec) DeleteBySubstring(s string) { - r.imageCache.deleteIfContains(s) -} - func (s *Spec) String() string { return "spec" } // TODO(bep) clean up below -func (r *Spec) newGenericResource(sourceFs afero.Fs, - targetPathBuilder func() page.TargetPaths, - osFileInfo os.FileInfo, - sourceFilename, - baseFilename string, - mediaType media.Type) *genericResource { - return r.newGenericResourceWithBase( - sourceFs, - nil, - nil, - targetPathBuilder, - osFileInfo, - sourceFilename, - baseFilename, - mediaType, - ) - -} - func (r *Spec) newGenericResourceWithBase( sourceFs afero.Fs, openReadSeekerCloser resource.OpenReadSeekCloser, diff --git a/resources/resource_test.go b/resources/resource_test.go index 7a0b8069d79..75ac94f79ce 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -34,7 +34,7 @@ func TestGenericResource(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) - r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo.css") @@ -48,11 +48,11 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := newTargetPaths("/foo") - r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo/foo.css") - c.Assert(r.Key(), qt.Equals, "/foo/foo.css") + c.Assert(r.Key(), qt.Equals, "foo/foo.css") c.Assert(r.ResourceType(), qt.Equals, "css") } @@ -105,10 +105,10 @@ func TestResourcesByType(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), - spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo.png", "logo.css", pngType), + newGenericResource(spec, nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} c.Assert(len(resources.ByType("css")), qt.Equals, 3) c.Assert(len(resources.ByType("image")), qt.Equals, 1) @@ -119,11 +119,11 @@ func TestResourcesGetByPrefix(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} c.Assert(resources.GetMatch("asdf*"), qt.IsNil) c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -148,14 +148,14 @@ func TestResourcesGetMatch(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), } c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -212,7 +212,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := resource.Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)} + resources := resource.Resources{newGenericResource(spec, nil, nil, nil, "/a/"+a100, a100, media.CSSType)} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -228,17 +228,17 @@ func benchResources(b *testing.B) resource.Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) } return resources @@ -266,7 +266,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } b.StartTimer() diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 8eacf7da4e6..3d7af38c23d 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -16,6 +16,8 @@ package htesting import ( "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -51,7 +53,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 0462f7ecdfc..3f5d6905f06 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -4,6 +4,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/cache/memcache" + "image" "io" "io/ioutil" @@ -90,7 +92,9 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + mc := memcache.New(memcache.Config{}) + + spec, err := NewSpec(s, filecaches, mc, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -129,7 +133,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir @@ -207,3 +211,22 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { t.Fatalf("Failed to write file: %s", err) } } + +func newGenericResource(r *Spec, sourceFs afero.Fs, + targetPathBuilder func() page.TargetPaths, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + nil, + nil, + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} diff --git a/resources/transform.go b/resources/transform.go index 98aee3c2a6f..5653b42b008 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -18,9 +18,12 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/resources/images/exif" @@ -42,6 +45,8 @@ var ( _ resource.ContentResource = (*resourceAdapter)(nil) _ resource.ReadSeekCloserResource = (*resourceAdapter)(nil) _ resource.Resource = (*resourceAdapter)(nil) + _ resource.Staler = (*resourceAdapter)(nil) + _ resource.Staler = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) _ resource.ResourceMetaProvider = (*resourceAdapter)(nil) @@ -60,12 +65,17 @@ func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResour if lazyPublish { po = &publishOnce{} } + + s := &resourceOrigin{} + return &resourceAdapter{ resourceTransformations: &resourceTransformations{}, + resourceOrigin: s, resourceAdapterInner: &resourceAdapterInner{ - spec: spec, - publishOnce: po, - target: target, + spec: spec, + resourceOrigin: s, + publishOnce: po, + target: target, }, } } @@ -150,6 +160,11 @@ type publishOnce struct { type resourceAdapter struct { commonResource + + // This state is carried over into any clone of this adapter + // (when passed through a Hugo pipe). + *resourceOrigin + *resourceTransformations *resourceAdapterInner } @@ -188,8 +203,7 @@ func (r *resourceAdapter) Exif() (*exif.Exif, error) { } func (r *resourceAdapter) Key() string { - r.init(false, false) - return r.target.(resource.Identifier).Key() + return r.TransformationKey() } func (r *resourceAdapter) MediaType() media.Type { @@ -252,9 +266,10 @@ func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransfo } r.resourceAdapterInner = &resourceAdapterInner{ - spec: r.spec, - publishOnce: &publishOnce{}, - target: r.target, + spec: r.spec, + resourceOrigin: r.resourceOrigin, + publishOnce: &publishOnce{}, + target: r.target, } return &r, nil @@ -297,6 +312,57 @@ func (r *resourceAdapter) publish() { } func (r *resourceAdapter) TransformationKey() string { + r.transformationsKeyInit.Do(func() { + if len(r.transformations) == 0 { + r.transformationsKey = r.target.Key() + return + } + + var adder string + for _, tr := range r.transformations { + adder = adder + "_" + tr.Key().Value() + } + + key := r.target.Key() + adder = "_" + helpers.MD5String(adder) + + // Preserve any file extension if possible. + dotIdx := strings.LastIndex(key, ".") + if dotIdx == -1 { + key += adder + } else { + key = key[:dotIdx] + adder + key[dotIdx:] + } + + key = memcache.CleanKey(key) + r.transformationsKey = key + }) + + return r.transformationsKey +} + +// We changed the format of the resource cache keys in Hugo v0.75. +// To reduce the nois, especially on the theme site, we fall back to reading +// files on the old format. +// TODO(bep) eventually remove. +func (r *resourceAdapter) transformationKeyV074() string { + cleanKey := func(key string) string { + return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") + } + + resourceKeyPartition := func(filename string) string { + ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") + if ext == "" { + ext = "other" + } + return ext + } + + resourceCacheKey := func(filename string) string { + filename = filepath.ToSlash(filename) + return path.Join(resourceKeyPartition(filename), filename) + } + // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. var key string @@ -304,35 +370,33 @@ func (r *resourceAdapter) TransformationKey() string { key = key + "_" + tr.Key().Value() } - base := ResourceCacheKey(r.target.Key()) - return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key) -} + base := resourceCacheKey(r.target.RelPermalink()) + return cleanKey(base) + "_" + helpers.MD5String(key) -func (r *resourceAdapter) transform(publish, setContent bool) error { - cache := r.spec.ResourceCache +} +func (r *resourceAdapter) getOrTransform(publish, setContent bool) error { key := r.TransformationKey() + res, err := r.spec.ResourceCache.cache.GetOrCreate(key, func() memcache.Entry { + r, err := r.transform(key, publish, setContent) + return memcache.Entry{ + Value: r, + Err: err, + } + }) - cached, found := cache.get(key) - - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - return nil + if err != nil { + return err } - // Acquire a write lock for the named transformation. - cache.nlocker.Lock(key) - // Check the cache again. - cached, found = cache.get(key) - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - cache.nlocker.Unlock(key) - return nil - } + r.resourceAdapterInner = res.(*resourceAdapterInner) - defer cache.nlocker.Unlock(key) - defer cache.set(key, r.resourceAdapterInner) + return nil +} + +func (r *resourceAdapter) transform(key string, publish, setContent bool) (*resourceAdapterInner, error) { + cache := r.spec.ResourceCache b1 := bp.GetBuffer() b2 := bp.GetBuffer() defer bp.PutBuffer(b1) @@ -353,7 +417,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { contentrc, err := contentReadSeekerCloser(r.target) if err != nil { - return err + return nil, err } defer contentrc.Close() @@ -426,21 +490,25 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { } else { err = tr.Transform(tctx) if err != nil && err != herrors.ErrFeatureNotAvailable { - return newErr(err) + return nil, newErr(err) } if mayBeCachedOnDisk { tryFileCache = r.spec.BuildConfig.UseResourceCache(err) } if err != nil && !tryFileCache { - return newErr(err) + return nil, newErr(err) } } if tryFileCache { f := r.target.tryTransformedFileCache(key, updates) if f == nil { - return newErr(errors.Errorf("resource %q not found in file cache", key)) + keyOldFormat := r.transformationKeyV074() + f = r.target.tryTransformedFileCache(keyOldFormat, updates) + if f == nil { + return nil, newErr(errors.Errorf("resource %q not found in file cache", key)) + } } transformedContentr = f updates.sourceFs = cache.fileCache.Fs @@ -465,7 +533,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { if publish { publicw, err := r.target.openPublishFileForWriting(updates.targetPath) if err != nil { - return err + return nil, err } publishwriters = append(publishwriters, publicw) } @@ -475,7 +543,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { // Also write it to the cache fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata()) if err != nil { - return err + return nil, err } updates.sourceFilename = &fi.Name updates.sourceFs = cache.fileCache.Fs @@ -506,7 +574,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { publishw := hugio.NewMultiWriteCloser(publishwriters...) _, err = io.Copy(publishw, transformedContentr) if err != nil { - return err + return nil, err } publishw.Close() @@ -517,11 +585,11 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { newTarget, err := r.target.cloneWithUpdates(updates) if err != nil { - return err + return nil, err } r.target = newTarget - return nil + return r.resourceAdapterInner, nil } func (r *resourceAdapter) init(publish, setContent bool) { @@ -541,7 +609,7 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { r.publishOnce = nil } - r.transformationsErr = r.transform(publish, setContent) + r.transformationsErr = r.getOrTransform(publish, setContent) if r.transformationsErr != nil { if r.spec.ErrorSender != nil { r.spec.ErrorSender.SendError(r.transformationsErr) @@ -559,16 +627,24 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { type resourceAdapterInner struct { target transformableResource + *resourceOrigin + spec *Spec // Handles publishing (to /public) if needed. *publishOnce } +func (r *resourceAdapterInner) ResourceTarget() resource.Resource { + return r.target +} + type resourceTransformations struct { - transformationsInit sync.Once - transformationsErr error - transformations []ResourceTransformation + transformationsInit sync.Once + transformationsErr error + transformationsKeyInit sync.Once + transformationsKey string + transformations []ResourceTransformation } type transformableResource interface { @@ -603,7 +679,6 @@ func (u *transformationUpdate) toTransformedResourceMetadata() transformedResour } func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) { - u.targetPath = ctx.OutPath u.mediaType = ctx.OutMediaType u.data = ctx.Data u.targetPath = ctx.InPath diff --git a/tpl/openapi/openapi3/openapi3.go b/tpl/openapi/openapi3/openapi3.go index 7dfd2f6a797..a95e288e1a4 100644 --- a/tpl/openapi/openapi3/openapi3.go +++ b/tpl/openapi/openapi3/openapi3.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" kopenapi3 "github.com/getkin/kin-openapi/openapi3" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/resources/resource" @@ -29,22 +29,15 @@ import ( // New returns a new instance of the openapi3-namespaced template functions. func New(deps *deps.Deps) *Namespace { - // TODO1 consolidate when merging that "other branch" -- but be aware of the keys. - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - return &Namespace{ - cache: cache, + cache: deps.MemCache.GetOrCreatePartition("tpl/openapi3", memcache.ClearOnChange), deps: deps, } } // Namespace provides template functions for the "openapi3". type Namespace struct { - cache *namedmemcache.Cache + cache memcache.Getter deps *deps.Deps } @@ -55,21 +48,22 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg return nil, errors.New("no Key set in Resource") } - v, err := ns.cache.GetOrCreate(key, func() (interface{}, error) { + v, err := ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } + defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } s := &kopenapi3.Swagger{} @@ -80,12 +74,14 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg err = metadecoders.Default.UnmarshalTo(b, f, s) } if err != nil { - return nil, err + return memcache.Entry{Err: err} } err = kopenapi3.NewSwaggerLoader().ResolveRefsIn(s, nil) - return s, err + return memcache.Entry{Value: s, Err: err, StaleFunc: func() bool { + return resource.IsStaleAny(r) + }} }) if err != nil { diff --git a/tpl/transform/init_test.go b/tpl/transform/init_test.go index 47bd8a391e6..393b6b140d8 100644 --- a/tpl/transform/init_test.go +++ b/tpl/transform/init_test.go @@ -16,6 +16,8 @@ package transform import ( "testing" + "github.com/gohugoio/hugo/cache/memcache" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/htesting/hqt" @@ -28,7 +30,9 @@ func TestInit(t *testing.T) { var ns *internal.TemplateFuncsNamespace for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns = nsf(&deps.Deps{}) + ns = nsf(&deps.Deps{ + MemCache: memcache.New(memcache.Config{}), + }) if ns.Name == name { found = true break diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index b168d2a50d4..e4ea17beba2 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -18,7 +18,7 @@ import ( "html" "html/template" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" @@ -27,22 +27,19 @@ import ( // New returns a new instance of the transform-namespaced template functions. func New(deps *deps.Deps) *Namespace { - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - + if deps.MemCache == nil { + panic("must provide MemCache") + } return &Namespace{ - cache: cache, deps: deps, + cache: deps.MemCache.GetOrCreatePartition("tpl/transform", memcache.ClearOnChange), } } // Namespace provides template functions for the "transform" namespace. type Namespace struct { - cache *namedmemcache.Cache deps *deps.Deps + cache memcache.Getter } // Emojify returns a copy of s with all emoji codes replaced with actual emojis. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index b3f4206ff6b..1a14661f246 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -16,6 +16,8 @@ package transform import ( "html/template" + "github.com/gohugoio/hugo/cache/memcache" + "testing" "github.com/gohugoio/hugo/common/loggers" @@ -251,6 +253,7 @@ func newDeps(cfg config.Provider) *deps.Deps { return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), + MemCache: memcache.New(memcache.Config{}), ContentSpec: cs, } } diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index b606c870aa7..ea43f6d0ff9 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -17,6 +17,8 @@ import ( "io/ioutil" "strings" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/common/types" @@ -69,24 +71,28 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key += decoder.OptionsKey() } - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } - return decoder.Unmarshal(b, f) + v, err := decoder.Unmarshal(b, f) + + return memcache.Entry{Value: v, Err: err, StaleFunc: func() bool { + return resource.IsStaleAny(r) + }} }) } @@ -97,13 +103,15 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key := helpers.MD5String(dataStr) - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := decoder.FormatFromContentString(dataStr) if f == "" { - return nil, errors.New("unknown format") + return memcache.Entry{Err: errors.New("unknown format")} } - return decoder.Unmarshal([]byte(dataStr), f) + v, err := decoder.Unmarshal([]byte(dataStr), f) + + return memcache.Entry{Value: v, Err: err} }) } diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 183bdefd5bc..8796cef3724 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -145,7 +145,7 @@ a;b;c`, mime: media.CSVType}, map[string]interface{}{"DElimiter": ";", "Comment" {tstNoStringer{}, nil, false}, } { - ns.cache.Clear() + ns.deps.MemCache.Clear() var args []interface{}