From c6d29f7fac77944cccbd48bd5f2794fd3c2297ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 25 Jun 2020 11:32:42 +0200 Subject: [PATCH] Consolidate most memory into a LRU cache This commit also consolidates all (or the most important) memory caches in Hugo. Fixes #7425 --- cache/memcache/memcache.go | 241 ++++++++++++++++++ .../memcache_test.go} | 56 +++- cache/namedmemcache/named_cache.go | 79 ------ commands/hugo.go | 22 +- deps/deps.go | 20 +- go.mod | 2 + go.sum | 11 + helpers/general.go | 18 ++ hugolib/hugo_sites.go | 1 + hugolib/hugo_sites_build.go | 4 + hugolib/hugo_sites_build_test.go | 11 +- hugolib/resource_cache_test.go | 78 ++++++ hugolib/resource_chain_test.go | 2 +- resources/image_cache.go | 159 ++++++------ resources/post_publish.go | 5 +- resources/resource.go | 3 +- resources/resource_cache.go | 101 ++------ .../resource_factories/bundler/bundler.go | 3 +- resources/resource_factories/create/create.go | 14 +- resources/resource_spec.go | 31 +-- resources/transform.go | 59 ++--- tpl/transform/transform.go | 14 +- tpl/transform/transform_test.go | 3 + tpl/transform/unmarshal.go | 4 +- tpl/transform/unmarshal_test.go | 2 +- 25 files changed, 577 insertions(+), 366 deletions(-) create mode 100644 cache/memcache/memcache.go rename cache/{namedmemcache/named_cache_test.go => memcache/memcache_test.go} (52%) delete mode 100644 cache/namedmemcache/named_cache.go create mode 100644 hugolib/resource_cache_test.go diff --git a/cache/memcache/memcache.go b/cache/memcache/memcache.go new file mode 100644 index 00000000000..f46d0917c6c --- /dev/null +++ b/cache/memcache/memcache.go @@ -0,0 +1,241 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memcache provides the core memory cache used in Hugo. +package memcache + +import ( + "fmt" + "runtime" + "sync/atomic" + "time" + + "github.com/BurntSushi/locker" + "github.com/gohugoio/hugo/helpers" + "github.com/karlseguin/ccache" + "github.com/pbnjay/memory" +) + +const ( + gigabyte = 1 << 30 +) + +// Cache configures a cache. +type Cache struct { + conf Config + cache *ccache.LayeredCache + + ttl time.Duration + nlocker *locker.Locker + + stats *stats + stop func() +} + +type stats struct { + memstatsStart runtime.MemStats + memstatsCurrent runtime.MemStats + maxSize int64 + + // This is an estimated/best guess value. TODO1 env factor. + availableMemory uint64 + + numItems uint64 +} + +func (s *stats) isLowOnMemory() bool { + return s.memstatsCurrent.Alloc > s.availableMemory +} + +func (s *stats) newMaxSize() int64 { + s.maxSize = s.maxSize / 2 + if s.maxSize < 20 { + s.maxSize = 20 + } + return s.maxSize +} + +func (s *stats) incr(i int) { + atomic.AddUint64(&s.numItems, uint64(i)) +} + +func (s *stats) decr(i int) { + atomic.AddUint64(&s.numItems, ^uint64(i-1)) +} + +type cacheEntry struct { + size int64 + value interface{} + err error +} + +func (c cacheEntry) Size() int64 { + return c.size +} + +type Config struct { + CheckInterval time.Duration + MaxSize int64 + ItemsToPrune uint32 + TTL time.Duration +} + +// New creates a new cache. +func New(conf Config) *Cache { + if conf.TTL == 0 { + conf.TTL = time.Second * 33 + } + if conf.CheckInterval == 0 { + conf.CheckInterval = time.Second * 2 + } + if conf.MaxSize == 0 { + conf.MaxSize = 1000 + } + if conf.ItemsToPrune == 0 { + conf.ItemsToPrune = 200 + } + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + var availableMemory uint64 + + // The total memory does not exclude memory used by other processes. + // For now, let's say that Hugo can use a fraction of it. + total := memory.TotalMemory() + if total != 0 { + availableMemory = total / 4 + } else { + availableMemory = 2 * gigabyte + } + + stats := &stats{ + memstatsStart: m, + maxSize: conf.MaxSize, + availableMemory: availableMemory, + } + + if stats.isLowOnMemory() { + conf.MaxSize = stats.newMaxSize() + } + + c := &Cache{ + conf: conf, + cache: ccache.Layered(ccache.Configure().MaxSize(conf.MaxSize).ItemsToPrune(conf.ItemsToPrune)), + ttl: conf.TTL, + stats: stats, + nlocker: locker.NewLocker(), + } + + c.stop = c.start() + + return c +} + +func (c *Cache) start() func() { + ticker := time.NewTicker(c.conf.CheckInterval) + quit := make(chan struct{}) + + checkAndAdjustMaxSize := func() { + var m runtime.MemStats + cacheDropped := c.GetDropped() + c.stats.decr(cacheDropped) + + runtime.ReadMemStats(&m) + c.stats.memstatsCurrent = m + if c.stats.isLowOnMemory() { + c.cache.SetMaxSize(c.stats.newMaxSize()) + } + + fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) + + } + go func() { + for { + select { + case <-ticker.C: + checkAndAdjustMaxSize() + case <-quit: + ticker.Stop() + return + } + } + }() + + return func() { + close(quit) + } +} + +// Clear clears the cache state. +// This method is not thread safe. +func (c *Cache) Clear() { + c.nlocker = locker.NewLocker() + c.cache.Clear() +} + +func (c *Cache) Has(primary, secondary string) bool { + return c.cache.Get(primary, secondary) != nil +} + +func (c *Cache) Get(primary, secondary string) (interface{}, bool) { + v := c.cache.Get(primary, secondary) + if v == nil { + return nil, false + } + return v.Value(), true +} + +func (c *Cache) DeleteAll(primary string) bool { + return c.cache.DeleteAll(primary) +} + +func (c *Cache) Stop() { + c.stop() + c.cache.Stop() +} + +func (c *Cache) GetDropped() int { + return c.cache.GetDropped() +} + +// GetOrCreate tries to get the value with the given cache keys, if not found +// create will be called and cached. +// This method is thread safe. +func (c *Cache) GetOrCreate(primary, secondary string, create func() (interface{}, error)) (interface{}, error) { + if v := c.cache.Get(primary, secondary); v != nil { + entry := v.Value().(cacheEntry) + return entry.value, entry.err + } + + // The provided create function may be a relatively time consuming operation, + // and there will in the commmon case be concurrent requests for the same key'd + // resource, so make sure we pause these until the result is ready. + key := primary + secondary + c.nlocker.Lock(key) + defer c.nlocker.Unlock(key) + + // Try again. + if v := c.cache.Get(primary, secondary); v != nil { + entry := v.Value().(cacheEntry) + return entry.value, entry.err + } + + // Create it and store it in cache. + value, err := create() + + c.cache.Set(primary, secondary, cacheEntry{value: value, err: err, size: 1}, c.ttl) + c.stats.incr(1) + + return value, err +} diff --git a/cache/namedmemcache/named_cache_test.go b/cache/memcache/memcache_test.go similarity index 52% rename from cache/namedmemcache/named_cache_test.go rename to cache/memcache/memcache_test.go index 9feddb11f2a..e5fec0a0510 100644 --- a/cache/namedmemcache/named_cache_test.go +++ b/cache/memcache/memcache_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2020 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,21 +11,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package namedmemcache +package memcache import ( "fmt" "sync" "testing" + "time" qt "github.com/frankban/quicktest" ) -func TestNamedCache(t *testing.T) { +func TesCache(t *testing.T) { t.Parallel() c := qt.New(t) - cache := New() + cache := New(Config{}) counter := 0 create := func() (interface{}, error) { @@ -34,29 +35,29 @@ func TestNamedCache(t *testing.T) { } for i := 0; i < 5; i++ { - v1, err := cache.GetOrCreate("a1", create) + v1, err := cache.GetOrCreate("a", "a1", create) c.Assert(err, qt.IsNil) c.Assert(v1, qt.Equals, 1) - v2, err := cache.GetOrCreate("a2", create) + v2, err := cache.GetOrCreate("a", "a2", create) c.Assert(err, qt.IsNil) c.Assert(v2, qt.Equals, 2) } cache.Clear() - v3, err := cache.GetOrCreate("a2", create) + v3, err := cache.GetOrCreate("a", "a2", create) c.Assert(err, qt.IsNil) c.Assert(v3, qt.Equals, 3) } -func TestNamedCacheConcurrent(t *testing.T) { +func TestCacheConcurrent(t *testing.T) { t.Parallel() c := qt.New(t) var wg sync.WaitGroup - cache := New() + cache := New(Config{}) create := func(i int) func() (interface{}, error) { return func() (interface{}, error) { @@ -70,7 +71,7 @@ func TestNamedCacheConcurrent(t *testing.T) { defer wg.Done() for j := 0; j < 100; j++ { id := fmt.Sprintf("id%d", j) - v, err := cache.GetOrCreate(id, create(j)) + v, err := cache.GetOrCreate("a", id, create(j)) c.Assert(err, qt.IsNil) c.Assert(v, qt.Equals, j) } @@ -78,3 +79,38 @@ func TestNamedCacheConcurrent(t *testing.T) { } wg.Wait() } + +func TestCacheMemStats(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{ + ItemsToPrune: 10, + CheckInterval: 500 * time.Millisecond, + }) + + s := cache.stats + + c.Assert(s.memstatsStart.Alloc > 0, qt.Equals, true) + c.Assert(s.memstatsCurrent.Alloc, qt.Equals, uint64(0)) + c.Assert(s.availableMemory > 0, qt.Equals, true) + c.Assert(s.numItems, qt.Equals, uint64(0)) + + counter := 0 + create := func() (interface{}, error) { + counter++ + return counter, nil + } + + for i := 1; i <= 20; i++ { + _, err := cache.GetOrCreate("a", fmt.Sprintf("b%d", i), create) + c.Assert(err, qt.IsNil) + } + + c.Assert(s.numItems, qt.Equals, uint64(20)) + cache.cache.SetMaxSize(10) + time.Sleep(time.Millisecond * 600) + c.Assert(int(s.numItems), qt.Equals, 10) + + c.Assert(s.memstatsCurrent.Alloc > 0, qt.Equals, true) +} diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go deleted file mode 100644 index d8c229a013b..00000000000 --- a/cache/namedmemcache/named_cache.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package namedmemcache provides a memory cache with a named lock. This is suitable -// for situations where creating the cached resource can be time consuming or otherwise -// resource hungry, or in situations where a "once only per key" is a requirement. -package namedmemcache - -import ( - "sync" - - "github.com/BurntSushi/locker" -) - -// Cache holds the cached values. -type Cache struct { - nlocker *locker.Locker - cache map[string]cacheEntry - mu sync.RWMutex -} - -type cacheEntry struct { - value interface{} - err error -} - -// New creates a new cache. -func New() *Cache { - return &Cache{ - nlocker: locker.NewLocker(), - cache: make(map[string]cacheEntry), - } -} - -// Clear clears the cache state. -func (c *Cache) Clear() { - c.mu.Lock() - defer c.mu.Unlock() - - c.cache = make(map[string]cacheEntry) - c.nlocker = locker.NewLocker() - -} - -// GetOrCreate tries to get the value with the given cache key, if not found -// create will be called and cached. -// This method is thread safe. It also guarantees that the create func for a given -// key is invoced only once for this cache. -func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) { - c.mu.RLock() - entry, found := c.cache[key] - c.mu.RUnlock() - - if found { - return entry.value, entry.err - } - - c.nlocker.Lock(key) - defer c.nlocker.Unlock(key) - - // Create it. - value, err := create() - - c.mu.Lock() - c.cache[key] = cacheEntry{value: value, err: err} - c.mu.Unlock() - - return value, err -} diff --git a/commands/hugo.go b/commands/hugo.go index 5442c32d708..de311aaa646 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -433,8 +433,14 @@ func (c *commandeer) initMemTicker() func() { quit := make(chan struct{}) printMem := func() { var m runtime.MemStats + var cacheDropped int + h := c.hugo() + if h != nil && h.MemCache != nil { + cacheDropped = h.MemCache.GetDropped() + } + runtime.ReadMemStats(&m) - fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\n\n", formatByteCount(m.Alloc), formatByteCount(m.TotalAlloc), formatByteCount(m.Sys), m.NumGC) + fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) } @@ -1215,17 +1221,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { return name } - -func formatByteCount(b uint64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", - float64(b)/float64(div), "kMGTPE"[exp]) -} diff --git a/deps/deps.go b/deps/deps.go index 82a16ba5947..b54a57ca4c4 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -5,6 +5,8 @@ import ( "sync/atomic" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/cache/filecache" @@ -62,9 +64,12 @@ type Deps struct { // The configuration to use Cfg config.Provider `json:"-"` - // The file cache to use. + // The file caches to use. FileCaches filecache.Caches + // The memory cache to use. + MemCache *memcache.Cache + // The translation func to use Translate func(translationID string, args ...interface{}) string `json:"-"` @@ -158,6 +163,13 @@ type ResourceProvider interface { Clone(deps *Deps) error } +// Stop stops all running caches etc. +func (d *Deps) Stop() { + if d.MemCache != nil { + d.MemCache.Stop() + } +} + func (d *Deps) Tmpl() tpl.TemplateHandler { return d.tmpl } @@ -236,11 +248,12 @@ func New(cfg DepsCfg) (*Deps, error) { if err != nil { return nil, errors.WithMessage(err, "failed to create file caches from configuration") } + memCache := memcache.New(memcache.Config{}) errorHandler := &globalErrHandler{} buildState := &BuildState{} - resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) + resourceSpec, err := resources.NewSpec(ps, fileCaches, memCache, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -277,6 +290,7 @@ func New(cfg DepsCfg) (*Deps, error) { Language: cfg.Language, Site: cfg.Site, FileCaches: fileCaches, + MemCache: memCache, BuildStartListeners: &Listeners{}, BuildState: buildState, Timeout: time.Duration(timeoutms) * time.Millisecond, @@ -311,7 +325,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er // The resource cache is global so reuse. // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) + d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.MemCache, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 7a20d2789e3..7a3ac8e5bb8 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93 github.com/gorilla/websocket v1.4.1 github.com/jdkato/prose v1.1.1 + github.com/karlseguin/ccache v1.0.2-0.20200626122230-40275a30c888 github.com/kr/pretty v0.2.0 // indirect github.com/kyokomi/emoji v2.2.1+incompatible github.com/magefile/mage v1.9.0 @@ -36,6 +37,7 @@ require ( github.com/nicksnyder/go-i18n v1.10.0 github.com/niklasfasching/go-org v1.1.0 github.com/olekukonko/tablewriter v0.0.4 + github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 github.com/pelletier/go-toml v1.6.0 // indirect github.com/pkg/errors v0.9.1 github.com/rogpeppe/go-internal v1.5.1 diff --git a/go.sum b/go.sum index 4cc8f2271dc..bfb12d0eb85 100644 --- a/go.sum +++ b/go.sum @@ -212,6 +212,14 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karlseguin/ccache v1.0.1 h1:0gpC6z1qtv0cKmsi5Su5tTB6bJ2vm9bfOLACpDEB/Ro= +github.com/karlseguin/ccache v1.0.2-0.20200216035407-d9aec58960c7 h1:JLz65tG+cbiJ/Yj46FF3rmVmH61g6TI8YWq8p0AX5Zc= +github.com/karlseguin/ccache v1.0.2-0.20200216035407-d9aec58960c7/go.mod h1:bm6z3svDxOYoWqVvk2JmnwOr6dtrTru4/MmlXksuQxk= +github.com/karlseguin/ccache v1.0.2-0.20200626122230-40275a30c888 h1:Pgq9C5Cc/VUYV3Jm9JhzWNPzj5ghtCZ+0CgvtVRx4w4= +github.com/karlseguin/ccache v1.0.2-0.20200626122230-40275a30c888/go.mod h1:bm6z3svDxOYoWqVvk2JmnwOr6dtrTru4/MmlXksuQxk= +github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= +github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= +github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -273,6 +281,8 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 h1:MfIUBZ1bz7TgvQLVa/yPJZOGeKEgs6eTKUjz3zB4B+U= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= @@ -375,6 +385,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/helpers/general.go b/helpers/general.go index 80e303087a5..377cba65578 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -61,6 +61,24 @@ func FindAvailablePort() (*net.TCPAddr, error) { return nil, err } +// FormatByteCount pretty formats b. +func FormatByteCount(bc uint64) string { + const ( + Gigabyte = 1 << 30 + Megabyte = 1 << 20 + Kilobyte = 1 << 10 + ) + switch { + case bc > Gigabyte || -bc > Gigabyte: + return fmt.Sprintf("%.2f GB", float64(bc)/Gigabyte) + case bc > Megabyte || -bc > Megabyte: + return fmt.Sprintf("%.2f MB", float64(bc)/Megabyte) + case bc > Kilobyte || -bc > Kilobyte: + return fmt.Sprintf("%.2f KB", float64(bc)/Kilobyte) + } + return fmt.Sprintf("%d B", bc) +} + // InStringArray checks if a string is an element of a slice of strings // and returns a boolean value. func InStringArray(arr []string, el string) bool { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index ee0d5c56368..c76a1b19ca2 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -563,6 +563,7 @@ func (h *HugoSites) reset(config *BuildCfg) { } h.init.Reset() + h.MemCache.Clear() } // resetLogs resets the log counters etc. Used to do a new build on the same sites. diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 67ee10e0978..b1d94c4eed4 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -51,6 +51,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Make sure we don't trigger rebuilds in parallel. h.runningMu.Lock() defer h.runningMu.Unlock() + } else { + defer func() { + h.Stop() + }() } ctx, task := trace.NewTask(context.Background(), "Build") diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 8d0872bd5e0..c5c40c29b1d 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -417,15 +417,18 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { func TestMultiSitesRebuild(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 - // This leaktest seems to be a little bit shaky on Travis. - if !isCI() { - defer leaktest.CheckTimeout(t, 10*time.Second)() - } c := qt.New(t) b := newMultiSiteTestDefaultBuilder(t).Running().CreateSites().Build(BuildCfg{}) + defer b.H.Stop() + + // This leaktest seems to be a little bit shaky on Travis. + if !isCI() { + defer leaktest.CheckTimeout(t, 10*time.Second)() + } + sites := b.H.Sites fs := b.Fs diff --git a/hugolib/resource_cache_test.go b/hugolib/resource_cache_test.go new file mode 100644 index 00000000000..c151caade38 --- /dev/null +++ b/hugolib/resource_cache_test.go @@ -0,0 +1,78 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "testing" +) + +func TestResourceCacheMultihost(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false + +[Languages] +[Languages.en] +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + toLang := func(format, lang string) string { + return fmt.Sprintf(format, lang) + } + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + for _, lang := range []string{"en", "fr"} { + b.WithContent( + toLang("b1/index.%s.md", lang), toLang("---\ntitle: Bundle %s\n---", lang), + toLang("b1/json1.%s.json", lang), toLang("json1 %s", lang), + toLang("b1/json2.%s.json", lang), toLang("json2 %s", lang), + ) + } + + b.WithTemplates("index.html", ` + +{{ $b := site.GetPage "b1" }} +{{ $jsons := $b.Resources.Match "*.json" }} +{{ $myjsons := $jsons | resources.Concat "myjsons.json" }} +Bundle: {{ $b.Permalink }} +JSONS: {{ $myjsons.Content }} + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/fr/index.html", ` +Bundle: https://example.fr/b1/ + + JSONS: json1 enjson2 en +`) + + b.AssertFileContent("public/en/index.html", ` +Bundle: https://example.com/b1/ + + JSONS: json1 enjson2 en +`) + +} diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index c687ca3421b..e208d4d2ed2 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -350,7 +350,7 @@ Edited content. `) b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil) - b.H.ResourceSpec.ClearCaches() + b.H.MemCache.Clear() } } diff --git a/resources/image_cache.go b/resources/image_cache.go index 1888b457f59..98851024f83 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -18,7 +18,8 @@ import ( "io" "path/filepath" "strings" - "sync" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/images" @@ -30,20 +31,19 @@ type imageCache struct { pathSpec *helpers.PathSpec fileCache *filecache.Cache - - mu sync.RWMutex - store map[string]*resourceAdapter + memCache *memcache.Cache } func (c *imageCache) deleteIfContains(s string) { - c.mu.Lock() - defer c.mu.Unlock() - s = c.normalizeKeyBase(s) - for k := range c.store { - if strings.Contains(k, s) { - delete(c.store, k) - } - } + // TODO1 + /* c.mu.Lock() + defer c.mu.Unlock() + s = c.normalizeKeyBase(s) + for k := range c.store { + if strings.Contains(k, s) { + delete(c.store, k) + } + }*/ } // The cache key is a lowecase path with Unix style slashes and it always starts with @@ -56,12 +56,6 @@ func (c *imageCache) normalizeKeyBase(key string) string { return strings.Trim(strings.ToLower(filepath.ToSlash(key)), "/") } -func (c *imageCache) clear() { - c.mu.Lock() - defer c.mu.Unlock() - c.store = make(map[string]*resourceAdapter) -} - func (c *imageCache) getOrCreate( parent *imageResource, conf images.ImageConfig, createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) { @@ -69,99 +63,90 @@ func (c *imageCache) getOrCreate( memKey := parent.relTargetPathForRel(relTarget.path(), false, false, false) memKey = c.normalizeKey(memKey) - // For the file cache we want to generate and store it once if possible. - fileKeyPath := relTarget - if fi := parent.root.getFileInfo(); fi != nil { - fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) - } - fileKey := fileKeyPath.path() + v, err := c.memCache.GetOrCreate("TODO1", memKey, func() (interface{}, error) { + // For the file cache we want to generate and store it once if possible. + fileKeyPath := relTarget + if fi := parent.root.getFileInfo(); fi != nil { + fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) + } + fileKey := fileKeyPath.path() - // First check the in-memory store, then the disk. - c.mu.RLock() - cachedImage, found := c.store[memKey] - c.mu.RUnlock() + var img *imageResource - if found { - return cachedImage, nil - } + // These funcs are protected by a named lock. + // read clones the parent to its new name and copies + // the content to the destinations. + read := func(info filecache.ItemInfo, r io.ReadSeeker) error { + img = parent.clone(nil) + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - var img *imageResource + if err := img.InitConfig(r); err != nil { + return err + } - // These funcs are protected by a named lock. - // read clones the parent to its new name and copies - // the content to the destinations. - read := func(info filecache.ItemInfo, r io.ReadSeeker) error { - img = parent.clone(nil) - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) + r.Seek(0, 0) - if err := img.InitConfig(r); err != nil { - return err - } + w, err := img.openDestinationsForWriting() + if err != nil { + return err + } - r.Seek(0, 0) + if w == nil { + // Nothing to write. + return nil + } + + defer w.Close() + _, err = io.Copy(w, r) - w, err := img.openDestinationsForWriting() - if err != nil { return err } - if w == nil { - // Nothing to write. - return nil - } + // create creates the image and encodes it to the cache (w). + create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + defer w.Close() - defer w.Close() - _, err = io.Copy(w, r) + var conv image.Image + img, conv, err = createImage() + if err != nil { + return + } + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - return err - } + return img.EncodeTo(conf, conv, w) + } + + // Now look in the file cache. - // create creates the image and encodes it to the cache (w). - create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { - defer w.Close() + // The definition of this counter is not that we have processed that amount + // (e.g. resized etc.), it can be fetched from file cache, + // but the count of processed image variations for this site. + c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - var conv image.Image - img, conv, err = createImage() + _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { - return + return nil, err } - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) - return img.EncodeTo(conf, conv, w) - } + // The file is now stored in this cache. + img.setSourceFs(c.fileCache.Fs) - // Now look in the file cache. + imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - // The definition of this counter is not that we have processed that amount - // (e.g. resized etc.), it can be fetched from file cache, - // but the count of processed image variations for this site. - c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) + return imgAdapter, nil + }) - _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { return nil, err } + return v.(*resourceAdapter), nil - // The file is now stored in this cache. - img.setSourceFs(c.fileCache.Fs) - - c.mu.Lock() - if cachedImage, found = c.store[memKey]; found { - c.mu.Unlock() - return cachedImage, nil - } - - imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - c.store[memKey] = imgAdapter - c.mu.Unlock() - - return imgAdapter, nil } -func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { - return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)} +func newImageCache(fileCache *filecache.Cache, memCache *memcache.Cache, ps *helpers.PathSpec) *imageCache { + return &imageCache{fileCache: fileCache, memCache: memCache, pathSpec: ps} } diff --git a/resources/post_publish.go b/resources/post_publish.go index b2adfa5ce03..49b6fb82491 100644 --- a/resources/post_publish.go +++ b/resources/post_publish.go @@ -19,12 +19,13 @@ import ( ) type transformationKeyer interface { - TransformationKey() string + TransformationKey() (string, string) } // PostProcess wraps the given Resource for later processing. func (spec *Spec) PostProcess(r resource.Resource) (postpub.PostPublishedResource, error) { - key := r.(transformationKeyer).TransformationKey() + primary, secondary := r.(transformationKeyer).TransformationKey() + key := primary + secondary spec.postProcessMu.RLock() result, found := spec.PostProcessResources[key] spec.postProcessMu.RUnlock() diff --git a/resources/resource.go b/resources/resource.go index acdf2d744ec..3577d96b7c2 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -172,8 +172,6 @@ func (commonResource) Slice(in interface{}) (interface{}, error) { return nil, fmt.Errorf("type %T is not a Resource", v) } groups[i] = g - { - } } return groups, nil default: @@ -235,6 +233,7 @@ func (l *genericResource) Data() interface{} { } func (l *genericResource) Key() string { + // TODO1 return l.RelPermalink() } diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 47822a7f506..d16ce2d5589 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -21,6 +21,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs/glob" @@ -28,8 +30,6 @@ import ( "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/cache/filecache" - - "github.com/BurntSushi/locker" ) const ( @@ -42,20 +42,18 @@ type ResourceCache struct { sync.RWMutex - // Either resource.Resource or resource.Resources. - cache map[string]interface{} + // Memory cache with either + // resource.Resource or resource.Resources. + cache *memcache.Cache fileCache *filecache.Cache - - // Provides named resource locks. - nlocker *locker.Locker } // ResourceCacheKey converts the filename into the format used in the resource // cache. -func ResourceCacheKey(filename string) string { +func ResourceCacheKey(filename string) (string, string) { filename = filepath.ToSlash(filename) - return path.Join(resourceKeyPartition(filename), filename) + return resourceKeyPartition(filename), filename } func resourceKeyPartition(filename string) string { @@ -123,26 +121,22 @@ func ResourceKeyContainsAny(key string, partitions []string) bool { return false } -func newResourceCache(rs *Spec) *ResourceCache { +func newResourceCache(rs *Spec, memCache *memcache.Cache) *ResourceCache { return &ResourceCache{ rs: rs, fileCache: rs.FileCaches.AssetsCache(), - cache: make(map[string]interface{}), - nlocker: locker.NewLocker(), + cache: memCache, } } -func (c *ResourceCache) clear() { - c.Lock() - defer c.Unlock() - - c.cache = make(map[string]interface{}) - c.nlocker = locker.NewLocker() +func (c *ResourceCache) Stop() { + c.cache.Stop() } func (c *ResourceCache) Contains(key string) bool { + // TODO1 key = c.cleanKey(filepath.ToSlash(key)) - _, found := c.get(key) + _, found := c.get("foo", key) return found } @@ -150,15 +144,12 @@ func (c *ResourceCache) cleanKey(key string) string { return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") } -func (c *ResourceCache) get(key string) (interface{}, bool) { - c.RLock() - defer c.RUnlock() - r, found := c.cache[key] - return r, found +func (c *ResourceCache) get(primary, secondary string) (interface{}, bool) { + return c.cache.Get(primary, secondary) } -func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, error)) (resource.Resource, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) +func (c *ResourceCache) GetOrCreate(primary, secondary string, f func() (resource.Resource, error)) (resource.Resource, error) { + r, err := c.cache.GetOrCreate(primary, secondary, func() (interface{}, error) { return f() }) if r == nil || err != nil { return nil, err } @@ -166,43 +157,13 @@ func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, err } func (c *ResourceCache) GetOrCreateResources(key string, f func() (resource.Resources, error)) (resource.Resources, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) + r, err := c.cache.GetOrCreate(CACHE_OTHER, key, func() (interface{}, error) { return f() }) if r == nil || err != nil { return nil, err } return r.(resource.Resources), nil } -func (c *ResourceCache) getOrCreate(key string, f func() (interface{}, error)) (interface{}, error) { - key = c.cleanKey(key) - // First check in-memory cache. - r, found := c.get(key) - if found { - return r, nil - } - // This is a potentially long running operation, so get a named lock. - c.nlocker.Lock(key) - - // Double check in-memory cache. - r, found = c.get(key) - if found { - c.nlocker.Unlock(key) - return r, nil - } - - defer c.nlocker.Unlock(key) - - r, err := f() - if err != nil { - return nil, err - } - - c.set(key, r) - - return r, nil - -} - func (c *ResourceCache) getFilenames(key string) (string, string) { filenameMeta := key + ".json" filenameContent := key + ".content" @@ -256,42 +217,26 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) } -func (c *ResourceCache) set(key string, r interface{}) { - c.Lock() - defer c.Unlock() - c.cache[key] = r -} - func (c *ResourceCache) DeletePartitions(partitions ...string) { partitionsSet := map[string]bool{ // Always clear out the resources not matching any partition. - "other": true, + CACHE_OTHER: true, } for _, p := range partitions { partitionsSet[p] = true } + // TODO1 move this method? if partitionsSet[CACHE_CLEAR_ALL] { - c.clear() + c.cache.Clear() return } c.Lock() defer c.Unlock() - for k := range c.cache { - clear := false - for p := range partitionsSet { - if strings.Contains(k, p) { - // There will be some false positive, but that's fine. - clear = true - break - } - } - - if clear { - delete(c.cache, k) - } + for p := range partitionsSet { + c.cache.DeleteAll(p) } } diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index 1ea92bea397..a58d037bac3 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -17,7 +17,6 @@ package bundler import ( "fmt" "io" - "path" "path/filepath" "github.com/gohugoio/hugo/common/hugio" @@ -82,7 +81,7 @@ func (r *multiReadSeekCloser) Close() error { // Concat concatenates the list of Resource objects. func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resource, error) { // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) { var resolvedm media.Type // The given set of resources must be of the same Media Type. diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 4ac20d36e5a..c9883bf5c36 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -18,7 +18,6 @@ package create import ( "path" "path/filepath" - "strings" "github.com/gohugoio/hugo/hugofs/glob" @@ -43,7 +42,8 @@ func New(rs *resources.Spec) *Client { // Get creates a new Resource by opening the given filename in the assets filesystem. func (c *Client) Get(filename string) (resource.Resource, error) { filename = filepath.Clean(filename) - return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(filename), func() (resource.Resource, error) { + primary, secondary := resources.ResourceCacheKey(filename) + return c.rs.ResourceCache.GetOrCreate(primary, secondary, func() (resource.Resource, error) { return c.rs.New(resources.ResourceSourceDescriptor{ Fs: c.rs.BaseFs.Assets.Fs, LazyPublish: true, @@ -74,13 +74,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro name = "__match" } - pattern = glob.NormalizePath(pattern) - partitions := glob.FilterGlobParts(strings.Split(pattern, "/")) - if len(partitions) == 0 { - partitions = []string{resources.CACHE_OTHER} - } - key := path.Join(name, path.Join(partitions...)) - key = path.Join(key, pattern) + key := path.Join(name, glob.NormalizePath(pattern)) return c.rs.ResourceCache.GetOrCreateResources(key, func() (resource.Resources, error) { var res resource.Resources @@ -116,7 +110,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) { return c.rs.New( resources.ResourceSourceDescriptor{ Fs: c.rs.FileCaches.AssetsCache().Fs, diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 81eed2f0203..b150b480a2a 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/config" @@ -46,6 +48,7 @@ import ( func NewSpec( s *helpers.PathSpec, fileCaches filecache.Caches, + memCache *memcache.Cache, incr identity.Incrementer, logger *loggers.Logger, errorHandler herrors.ErrorSender, @@ -89,11 +92,11 @@ func NewSpec( PostProcessResources: make(map[string]postpub.PostPublishedResource), imageCache: newImageCache( fileCaches.ImageCache(), - + memCache, s, )} - rs.ResourceCache = newResourceCache(rs) + rs.ResourceCache = newResourceCache(rs, memCache) return rs, nil @@ -129,29 +132,7 @@ func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { return r.newResourceFor(fd) } -func (r *Spec) CacheStats() string { - r.imageCache.mu.RLock() - defer r.imageCache.mu.RUnlock() - - s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) - - count := 0 - for k := range r.imageCache.store { - if count > 5 { - break - } - s += "\n" + k - count++ - } - - return s -} - -func (r *Spec) ClearCaches() { - r.imageCache.clear() - r.ResourceCache.clear() -} - +// TODO1 func (r *Spec) DeleteBySubstring(s string) { r.imageCache.deleteIfContains(s) } diff --git a/resources/transform.go b/resources/transform.go index 98aee3c2a6f..07a0cd7f8f4 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -296,7 +296,7 @@ func (r *resourceAdapter) publish() { } -func (r *resourceAdapter) TransformationKey() string { +func (r *resourceAdapter) TransformationKey() (string, string) { // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. var key string @@ -304,35 +304,28 @@ func (r *resourceAdapter) TransformationKey() string { key = key + "_" + tr.Key().Value() } - base := ResourceCacheKey(r.target.Key()) - return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key) + primary, secondary := ResourceCacheKey(r.target.Key()) + return primary, r.spec.ResourceCache.cleanKey(secondary) + "_" + helpers.MD5String(key) } -func (r *resourceAdapter) transform(publish, setContent bool) error { - cache := r.spec.ResourceCache - - key := r.TransformationKey() - - cached, found := cache.get(key) +func (r *resourceAdapter) getOrTransform(publish, setContent bool) error { + primary, secondary := r.TransformationKey() + res, err := r.spec.ResourceCache.cache.GetOrCreate(primary, secondary, func() (interface{}, error) { + return r.transform2(secondary, publish, setContent) + }) - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - return nil + if err != nil { + return err } - // Acquire a write lock for the named transformation. - cache.nlocker.Lock(key) - // Check the cache again. - cached, found = cache.get(key) - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - cache.nlocker.Unlock(key) - return nil - } + r.resourceAdapterInner = res.(*resourceAdapterInner) + + return nil - defer cache.nlocker.Unlock(key) - defer cache.set(key, r.resourceAdapterInner) +} +func (r *resourceAdapter) transform2(key string, publish, setContent bool) (*resourceAdapterInner, error) { + cache := r.spec.ResourceCache b1 := bp.GetBuffer() b2 := bp.GetBuffer() defer bp.PutBuffer(b1) @@ -353,7 +346,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { contentrc, err := contentReadSeekerCloser(r.target) if err != nil { - return err + return nil, err } defer contentrc.Close() @@ -426,21 +419,21 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { } else { err = tr.Transform(tctx) if err != nil && err != herrors.ErrFeatureNotAvailable { - return newErr(err) + return nil, newErr(err) } if mayBeCachedOnDisk { tryFileCache = r.spec.BuildConfig.UseResourceCache(err) } if err != nil && !tryFileCache { - return newErr(err) + return nil, newErr(err) } } if tryFileCache { f := r.target.tryTransformedFileCache(key, updates) if f == nil { - return newErr(errors.Errorf("resource %q not found in file cache", key)) + return nil, newErr(errors.Errorf("resource %q not found in file cache", key)) } transformedContentr = f updates.sourceFs = cache.fileCache.Fs @@ -465,7 +458,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { if publish { publicw, err := r.target.openPublishFileForWriting(updates.targetPath) if err != nil { - return err + return nil, err } publishwriters = append(publishwriters, publicw) } @@ -475,7 +468,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { // Also write it to the cache fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata()) if err != nil { - return err + return nil, err } updates.sourceFilename = &fi.Name updates.sourceFs = cache.fileCache.Fs @@ -506,7 +499,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { publishw := hugio.NewMultiWriteCloser(publishwriters...) _, err = io.Copy(publishw, transformedContentr) if err != nil { - return err + return nil, err } publishw.Close() @@ -517,11 +510,11 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { newTarget, err := r.target.cloneWithUpdates(updates) if err != nil { - return err + return nil, err } r.target = newTarget - return nil + return r.resourceAdapterInner, nil } func (r *resourceAdapter) init(publish, setContent bool) { @@ -541,7 +534,7 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { r.publishOnce = nil } - r.transformationsErr = r.transform(publish, setContent) + r.transformationsErr = r.getOrTransform(publish, setContent) if r.transformationsErr != nil { if r.spec.ErrorSender != nil { r.spec.ErrorSender.SendError(r.transformationsErr) diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index b168d2a50d4..f034478e1eb 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -18,8 +18,6 @@ import ( "html" "html/template" - "github.com/gohugoio/hugo/cache/namedmemcache" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/spf13/cast" @@ -27,22 +25,14 @@ import ( // New returns a new instance of the transform-namespaced template functions. func New(deps *deps.Deps) *Namespace { - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - return &Namespace{ - cache: cache, - deps: deps, + deps: deps, } } // Namespace provides template functions for the "transform" namespace. type Namespace struct { - cache *namedmemcache.Cache - deps *deps.Deps + deps *deps.Deps } // Emojify returns a copy of s with all emoji codes replaced with actual emojis. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index b3f4206ff6b..8c7374ea2f6 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -16,6 +16,8 @@ package transform import ( "html/template" + "github.com/gohugoio/hugo/cache/memcache" + "testing" "github.com/gohugoio/hugo/common/loggers" @@ -251,6 +253,7 @@ func newDeps(cfg config.Provider) *deps.Deps { return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), + MemCache: memcache.New(), ContentSpec: cs, } } diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index da06b6aa124..369136e648f 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -66,7 +66,7 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key += decoder.OptionsKey() } - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.deps.MemCache.GetOrCreate("TODO1", key, func() (interface{}, error) { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { return nil, errors.Errorf("MIME %q not supported", r.MediaType()) @@ -94,7 +94,7 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key := helpers.MD5String(dataStr) - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.deps.MemCache.GetOrCreate("TODO1", key, func() (interface{}, error) { f := decoder.FormatFromContentString(dataStr) if f == "" { return nil, errors.New("unknown format") diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 7b0caa07f05..8dcef6e6e06 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -145,7 +145,7 @@ a;b;c`, mime: media.CSVType}, map[string]interface{}{"DElimiter": ";", "Comment" {tstNoStringer{}, nil, false}, } { - ns.cache.Clear() + ns.deps.MemCache.Clear() var args []interface{}