From ac391f91a44d6aa053ad863e388e15c963e35729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 25 Jun 2020 11:32:42 +0200 Subject: [PATCH] Add central LRU cache that adjusts to available memory Hugo has always been a active user of in-memory caches, but before this commit we did nothing to control the memory usage. One failing example would be loading lots of big JSON data files and unmarshal them via `transform.Unmarshal`. This commit consolidates all these caches into one single LRU cache with an eviction strategy that also considers used vs. available memory. Hugo will try to limit its memory usage to 1/4 or total system memory, but this can be controlled with the `HUGO_MEMORYLIMIT` environment variable (a float value representing Gigabytes). A natural next step after this would be to use this cache for `.Content`. Fixes #7425 Fixes #7437 Fixes #7436 Updates #7544 --- cache/memcache/memcache.go | 459 ++++++++++++++++++ cache/memcache/memcache_test.go | 170 +++++++ cache/namedmemcache/named_cache.go | 79 --- cache/namedmemcache/named_cache_test.go | 80 --- commands/commands.go | 2 +- commands/hugo.go | 22 +- config/env.go | 35 ++ config/env_test.go | 2 + deps/deps.go | 20 +- .../en/getting-started/configuration.md | 9 + go.mod | 2 + go.sum | 102 ++-- helpers/general.go | 18 + hugofs/fileinfo.go | 5 + hugofs/rootmapping_fs.go | 1 + hugolib/content_map_page.go | 7 + hugolib/filesystems/basefs.go | 61 ++- hugolib/hugo_sites_build.go | 4 + hugolib/hugo_sites_build_test.go | 81 +++- hugolib/hugo_sites_multihost_test.go | 2 +- hugolib/pagebundler_test.go | 4 +- hugolib/pages_capture.go | 4 - hugolib/paths/paths.go | 6 +- hugolib/resource_chain_test.go | 2 +- hugolib/resource_change_test.go | 322 ++++++++++++ hugolib/site.go | 40 +- hugolib/testhelpers_test.go | 27 +- modules/client.go | 2 +- resources/image.go | 4 +- resources/image_cache.go | 166 +++---- resources/image_test.go | 2 +- resources/resource.go | 52 +- resources/resource/resourcetypes.go | 30 ++ resources/resource_cache.go | 229 +-------- resources/resource_cache_test.go | 58 --- .../resource_factories/bundler/bundler.go | 6 +- resources/resource_factories/create/create.go | 20 +- resources/resource_metadata_test.go | 12 +- resources/resource_spec.go | 53 +- resources/resource_test.go | 50 +- .../htesting/testhelpers.go | 4 +- resources/testhelpers_test.go | 27 +- resources/transform.go | 161 ++++-- tpl/openapi/openapi3/openapi3.go | 28 +- tpl/transform/init_test.go | 6 +- tpl/transform/transform.go | 15 +- tpl/transform/transform_test.go | 3 + tpl/transform/unmarshal.go | 24 +- tpl/transform/unmarshal_test.go | 2 +- 49 files changed, 1721 insertions(+), 799 deletions(-) create mode 100644 cache/memcache/memcache.go create mode 100644 cache/memcache/memcache_test.go delete mode 100644 cache/namedmemcache/named_cache.go delete mode 100644 cache/namedmemcache/named_cache_test.go create mode 100644 hugolib/resource_change_test.go delete mode 100644 resources/resource_cache_test.go diff --git a/cache/memcache/memcache.go b/cache/memcache/memcache.go new file mode 100644 index 00000000000..765a56403a5 --- /dev/null +++ b/cache/memcache/memcache.go @@ -0,0 +1,459 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memcache provides the core memory cache used in Hugo. +package memcache + +import ( + "math" + "path" + "regexp" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/resources/resource" + + "github.com/gohugoio/hugo/helpers" + + "github.com/BurntSushi/locker" + "github.com/karlseguin/ccache/v2" +) + +const ( + ClearOnRebuild ClearWhen = iota + ClearOnChange + ClearNever +) + +const ( + cacheVirtualRoot = "_root/" +) + +var ( + + // Consider a change in files matching this expression a "JS change". + isJSFileRe = regexp.MustCompile(`\.(js|ts|jsx|tsx)`) + + // Consider a change in files matching this expression a "CSS change". + isCSSFileRe = regexp.MustCompile(`\.(css|scss|sass)`) + + // These config files are tightly related to CSS editing, so consider + // a change to any of them a "CSS change". + isCSSConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`) +) + +const unknownExtension = "unkn" + +// New creates a new cache. +func New(conf Config) *Cache { + if conf.TTL == 0 { + conf.TTL = time.Second * 33 + } + if conf.CheckInterval == 0 { + conf.CheckInterval = time.Second * 2 + } + if conf.MaxSize == 0 { + conf.MaxSize = 100000 + } + if conf.ItemsToPrune == 0 { + conf.ItemsToPrune = 200 + } + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + stats := &stats{ + memstatsStart: m, + maxSize: conf.MaxSize, + availableMemory: config.GetMemoryLimit(), + } + + conf.MaxSize = stats.adjustMaxSize(nil) + + c := &Cache{ + conf: conf, + cache: ccache.Layered(ccache.Configure().MaxSize(conf.MaxSize).ItemsToPrune(conf.ItemsToPrune)), + getters: make(map[string]*getter), + ttl: conf.TTL, + stats: stats, + nlocker: locker.NewLocker(), + } + + c.stop = c.start() + + return c +} + +// CleanKey turns s into a format suitable for a cache key for this package. +// The key will be a Unix-styled path without any leading slash. +// If the input string does not contain any slash, a root will be prepended. +// If the input string does not contain any ".", a dummy file suffix will be appended. +// These are to make sure that they can effectively partake in the "cache cleaning" +// strategy used in server mode. +func CleanKey(s string) string { + s = path.Clean(helpers.ToSlashTrimLeading(s)) + if !strings.ContainsRune(s, '/') { + s = cacheVirtualRoot + s + } + if !strings.ContainsRune(s, '.') { + s += "." + unknownExtension + } + + return s +} + +// InsertKeyPathElement inserts the given element after the first '/' in key. +func InsertKeyPathElements(key string, elements ...string) string { + slashIdx := strings.Index(key, "/") + return key[:slashIdx] + "/" + path.Join(elements...) + key[slashIdx:] +} + +// Cache configures a cache. +type Cache struct { + mu sync.Mutex + getters map[string]*getter + + conf Config + cache *ccache.LayeredCache + + ttl time.Duration + nlocker *locker.Locker + + stats *stats + stopOnce sync.Once + stop func() +} + +// Clear clears the cache state. +// This method is not thread safe. +func (c *Cache) Clear() { + c.nlocker = locker.NewLocker() + for _, g := range c.getters { + g.c.DeleteAll(g.partition) + } +} + +// ClearOn clears all the caches given a eviction strategy and (optional) a +// change set. +// This method is not thread safe. +func (c *Cache) ClearOn(when ClearWhen, changeset ...string) { + for _, g := range c.getters { + if g.clearWhen == ClearNever { + continue + } + + if g.clearWhen == when { + // Clear all. + g.Clear() + return + } + + changeset = helpers.UniqueStrings(changeset) + shouldDelete := func(key string, e Entry) bool { + if e.ClearWhen == when || resource.IsStaleAny(e, e.Value) { + return true + } + + if len(changeset) == 0 { + return false + } + + for _, filename := range changeset { + path := CleanKey(filename) + + prefix, _ := splitBasePathAndExt(path) + if prefix == "" { + prefix = cacheVirtualRoot + } + + // Will clear out all files that share a common root path, + // e.g "styles" in "styles/main.css". + if strings.HasPrefix(key, prefix) { + return true + } + + switch v := e.Value.(type) { + case resource.MediaTypeProvider: + // Clear out all files sharing the same or related MIME type. + isJS := isJSFileRe.MatchString(path) + var isCSS bool + if !isJS { + isCSS = isCSSFileRe.MatchString(path) || isCSSConfigRe.MatchString(path) + } + + if isJS && isJSType(v.MediaType()) { + return true + } + + if isCSS && isCSSType(v.MediaType()) { + return true + } + + } + } + + // Keep it. + return false + } + + g.c.cache.DeleteFunc(g.partition, func(key string, item *ccache.Item) bool { + e := item.Value().(Entry) + if shouldDelete(key, e) { + resource.MarkStale(e.Value) + return true + } + return false + }) + + } +} + +func (c *Cache) DeleteAll(primary string) bool { + return c.cache.DeleteAll(primary) +} + +func (c *Cache) GetDropped() int { + return c.cache.GetDropped() +} + +func (c *Cache) GetOrCreatePartition(partition string, clearWhen ClearWhen) Getter { + c.mu.Lock() + defer c.mu.Unlock() + + g, found := c.getters[partition] + if found { + if g.clearWhen != clearWhen { + panic("GetOrCreatePartition called with the same partition but different clearing strategy.") + } + return g + } + + g = &getter{ + partition: partition, + c: c, + clearWhen: clearWhen, + } + + c.getters[partition] = g + + return g +} + +func (c *Cache) Stop() { + c.stopOnce.Do(func() { + c.stop() + c.cache.Stop() + }) +} + +func (c *Cache) start() func() { + ticker := time.NewTicker(c.conf.CheckInterval) + quit := make(chan struct{}) + + checkAndAdjustMaxSize := func() { + var m runtime.MemStats + cacheDropped := c.GetDropped() + c.stats.decr(cacheDropped) + + runtime.ReadMemStats(&m) + c.stats.memstatsCurrent = m + c.stats.adjustMaxSize(c.cache.SetMaxSize) + + //fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) + + } + go func() { + for { + select { + case <-ticker.C: + checkAndAdjustMaxSize() + case <-quit: + ticker.Stop() + return + } + } + }() + + return func() { + close(quit) + } +} + +// GetOrCreate tries to get the value with the given cache paths, if not found +// create will be called and the result cached. +// This method is thread safe. +func (c *Cache) getOrCreate(primary, secondary string, create func() Entry) (interface{}, error) { + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // The provided create function may be a relatively time consuming operation, + // and there will in the commmon case be concurrent requests for the same key'd + // resource, so make sure we pause these until the result is ready. + path := primary + secondary + c.nlocker.Lock(path) + defer c.nlocker.Unlock(path) + + // Try again. + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // Create it and store it in cache. + entry := create() + entry.size = 1 // For now. + if entry.Err != nil { + entry.ClearWhen = ClearOnRebuild + } + + c.cache.Set(primary, secondary, entry, c.ttl) + c.stats.incr(1) + + return entry.Value, entry.Err +} + +type ClearWhen int + +type Config struct { + CheckInterval time.Duration + MaxSize int64 + ItemsToPrune uint32 + TTL time.Duration + Running bool +} + +type Entry struct { + Value interface{} + size int64 + Err error + StaleFunc func() bool + ClearWhen +} + +func (e Entry) Size() int64 { + return e.size +} + +func (e Entry) IsStale() bool { + return e.StaleFunc != nil && e.StaleFunc() +} + +type Getter interface { + Clear() + GetOrCreate(path string, create func() Entry) (interface{}, error) +} + +type getter struct { + c *Cache + partition string + + clearWhen ClearWhen +} + +func (g *getter) Clear() { + g.c.DeleteAll(g.partition) +} + +func (g *getter) GetOrCreate(path string, create func() Entry) (interface{}, error) { + return g.c.getOrCreate(g.partition, path, create) +} + +type stats struct { + memstatsStart runtime.MemStats + memstatsCurrent runtime.MemStats + maxSize int64 + availableMemory uint64 + numItems uint64 +} + +func (s *stats) getNumItems() uint64 { + return atomic.LoadUint64(&s.numItems) +} + +func (s *stats) adjustMaxSize(setter func(size int64)) int64 { + newSize := int64(float64(s.maxSize) * s.resizeFactor()) + if newSize != s.maxSize && setter != nil { + setter(newSize) + } + return newSize +} + +func (s *stats) decr(i int) { + atomic.AddUint64(&s.numItems, ^uint64(i-1)) +} + +func (s *stats) incr(i int) { + atomic.AddUint64(&s.numItems, uint64(i)) +} + +func (s *stats) resizeFactor() float64 { + if s.memstatsCurrent.Alloc == 0 { + return 1.0 + } + return math.Floor(float64(s.availableMemory/s.memstatsCurrent.Alloc)*10) / 10 +} + +// Helpers to help eviction of related media types. +func isCSSType(m media.Type) bool { + tp := m.Type() + return tp == media.CSSType.Type() || tp == media.SASSType.Type() || tp == media.SCSSType.Type() +} + +func isJSType(m media.Type) bool { + tp := m.Type() + return tp == media.JavascriptType.Type() || tp == media.TypeScriptType.Type() || tp == media.JSXType.Type() || tp == media.TSXType.Type() +} + +func keyValid(s string) bool { + if len(s) < 5 { + return false + } + if strings.ContainsRune(s, '\\') { + return false + } + if strings.HasPrefix(s, "/") { + return false + } + if !strings.ContainsRune(s, '/') { + return false + } + + dotIdx := strings.Index(s, ".") + if dotIdx == -1 || dotIdx == len(s)-1 { + return false + } + + return true +} + +// This assumes a valid key path. +func splitBasePathAndExt(path string) (string, string) { + dotIdx := strings.LastIndex(path, ".") + ext := path[dotIdx+1:] + slashIdx := strings.Index(path, "/") + + return path[:slashIdx], ext +} diff --git a/cache/memcache/memcache_test.go b/cache/memcache/memcache_test.go new file mode 100644 index 00000000000..0a5cf76f9cc --- /dev/null +++ b/cache/memcache/memcache_test.go @@ -0,0 +1,170 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memcache + +import ( + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + qt "github.com/frankban/quicktest" +) + +func TestCache(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{}) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + a := cache.GetOrCreatePartition("a", ClearNever) + + for i := 0; i < 5; i++ { + v1, err := a.GetOrCreate("a1", create) + c.Assert(err, qt.IsNil) + c.Assert(v1, qt.Equals, 1) + v2, err := a.GetOrCreate("a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v2, qt.Equals, 2) + } + + cache.Clear() + + v3, err := a.GetOrCreate("a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v3, qt.Equals, 3) +} + +func TestCacheConcurrent(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + var wg sync.WaitGroup + + cache := New(Config{}) + + create := func(i int) func() Entry { + return func() Entry { + return Entry{Value: i} + } + } + + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + id := fmt.Sprintf("id%d", j) + v, err := cache.getOrCreate("a", id, create(j)) + c.Assert(err, qt.IsNil) + c.Assert(v, qt.Equals, j) + } + }() + } + wg.Wait() +} + +func TestCacheMemStats(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{ + ItemsToPrune: 10, + CheckInterval: 500 * time.Millisecond, + }) + + s := cache.stats + + c.Assert(s.memstatsStart.Alloc > 0, qt.Equals, true) + c.Assert(s.memstatsCurrent.Alloc, qt.Equals, uint64(0)) + c.Assert(s.availableMemory > 0, qt.Equals, true) + c.Assert(s.numItems, qt.Equals, uint64(0)) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + for i := 1; i <= 20; i++ { + _, err := cache.getOrCreate("a", fmt.Sprintf("b%d", i), create) + c.Assert(err, qt.IsNil) + } + + c.Assert(s.getNumItems(), qt.Equals, uint64(20)) + cache.cache.SetMaxSize(10) + time.Sleep(time.Millisecond * 600) + c.Assert(int(s.getNumItems()), qt.Equals, 10) + +} + +func TestSplitBasePathAndExt(t *testing.T) { + t.Parallel() + c := qt.New(t) + + tests := []struct { + path string + a string + b string + }{ + {"a/b.json", "a", "json"}, + {"a/b/c/d.json", "a", "json"}, + } + for i, this := range tests { + msg := qt.Commentf("test %d", i) + a, b := splitBasePathAndExt(this.path) + + c.Assert(a, qt.Equals, this.a, msg) + c.Assert(b, qt.Equals, this.b, msg) + } + +} + +func TestCleanKey(t *testing.T) { + c := qt.New(t) + + c.Assert(CleanKey(filepath.FromSlash("a/b/c.js")), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a//b////c.js"), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a.js"), qt.Equals, "_root/a.js") + c.Assert(CleanKey("b/a"), qt.Equals, "b/a.unkn") + +} + +func TestKeyValid(t *testing.T) { + c := qt.New(t) + + c.Assert(keyValid("a/b.j"), qt.Equals, true) + c.Assert(keyValid("a/b."), qt.Equals, false) + c.Assert(keyValid("a/b"), qt.Equals, false) + c.Assert(keyValid("/a/b.txt"), qt.Equals, false) + c.Assert(keyValid("a\\b.js"), qt.Equals, false) + +} + +func TestInsertKeyPathElement(t *testing.T) { + c := qt.New(t) + + c.Assert(InsertKeyPathElements("a/b.j", "en"), qt.Equals, "a/en/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "en", "foo"), qt.Equals, "a/en/foo/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "", "foo"), qt.Equals, "a/foo/b.j") + +} diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go deleted file mode 100644 index d8c229a013b..00000000000 --- a/cache/namedmemcache/named_cache.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package namedmemcache provides a memory cache with a named lock. This is suitable -// for situations where creating the cached resource can be time consuming or otherwise -// resource hungry, or in situations where a "once only per key" is a requirement. -package namedmemcache - -import ( - "sync" - - "github.com/BurntSushi/locker" -) - -// Cache holds the cached values. -type Cache struct { - nlocker *locker.Locker - cache map[string]cacheEntry - mu sync.RWMutex -} - -type cacheEntry struct { - value interface{} - err error -} - -// New creates a new cache. -func New() *Cache { - return &Cache{ - nlocker: locker.NewLocker(), - cache: make(map[string]cacheEntry), - } -} - -// Clear clears the cache state. -func (c *Cache) Clear() { - c.mu.Lock() - defer c.mu.Unlock() - - c.cache = make(map[string]cacheEntry) - c.nlocker = locker.NewLocker() - -} - -// GetOrCreate tries to get the value with the given cache key, if not found -// create will be called and cached. -// This method is thread safe. It also guarantees that the create func for a given -// key is invoced only once for this cache. -func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) { - c.mu.RLock() - entry, found := c.cache[key] - c.mu.RUnlock() - - if found { - return entry.value, entry.err - } - - c.nlocker.Lock(key) - defer c.nlocker.Unlock(key) - - // Create it. - value, err := create() - - c.mu.Lock() - c.cache[key] = cacheEntry{value: value, err: err} - c.mu.Unlock() - - return value, err -} diff --git a/cache/namedmemcache/named_cache_test.go b/cache/namedmemcache/named_cache_test.go deleted file mode 100644 index 9feddb11f2a..00000000000 --- a/cache/namedmemcache/named_cache_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package namedmemcache - -import ( - "fmt" - "sync" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestNamedCache(t *testing.T) { - t.Parallel() - c := qt.New(t) - - cache := New() - - counter := 0 - create := func() (interface{}, error) { - counter++ - return counter, nil - } - - for i := 0; i < 5; i++ { - v1, err := cache.GetOrCreate("a1", create) - c.Assert(err, qt.IsNil) - c.Assert(v1, qt.Equals, 1) - v2, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v2, qt.Equals, 2) - } - - cache.Clear() - - v3, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v3, qt.Equals, 3) -} - -func TestNamedCacheConcurrent(t *testing.T) { - t.Parallel() - - c := qt.New(t) - - var wg sync.WaitGroup - - cache := New() - - create := func(i int) func() (interface{}, error) { - return func() (interface{}, error) { - return i, nil - } - } - - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - id := fmt.Sprintf("id%d", j) - v, err := cache.GetOrCreate(id, create(j)) - c.Assert(err, qt.IsNil) - c.Assert(v, qt.Equals, j) - } - }() - } - wg.Wait() -} diff --git a/commands/commands.go b/commands/commands.go index ce5f0ff7d97..bbbbc5aba4d 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -301,7 +301,7 @@ func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) { cmd.Flags().BoolP("path-warnings", "", false, "print warnings on duplicate target paths etc.") cmd.Flags().StringVarP(&cc.cpuprofile, "profile-cpu", "", "", "write cpu profile to `file`") cmd.Flags().StringVarP(&cc.memprofile, "profile-mem", "", "", "write memory profile to `file`") - cmd.Flags().BoolVarP(&cc.printm, "print-mem", "", false, "print memory usage to screen at intervals") + cmd.Flags().BoolVarP(&cc.printm, "printMem", "", false, "print memory usage to screen at intervals") cmd.Flags().StringVarP(&cc.mutexprofile, "profile-mutex", "", "", "write Mutex profile to `file`") cmd.Flags().StringVarP(&cc.traceprofile, "trace", "", "", "write trace to `file` (not useful in general)") diff --git a/commands/hugo.go b/commands/hugo.go index 058f1ec7ce7..d30e2ea4700 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -427,8 +427,14 @@ func (c *commandeer) initMemTicker() func() { quit := make(chan struct{}) printMem := func() { var m runtime.MemStats + var cacheDropped int + h := c.hugo() + if h != nil && h.MemCache != nil { + cacheDropped = h.MemCache.GetDropped() + } + runtime.ReadMemStats(&m) - fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\n\n", formatByteCount(m.Alloc), formatByteCount(m.TotalAlloc), formatByteCount(m.Sys), m.NumGC) + fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\nConfiguredMemoryLimit = %v\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped, helpers.FormatByteCount(config.GetMemoryLimit())) } @@ -1209,17 +1215,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { return name } - -func formatByteCount(b uint64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", - float64(b)/float64(div), "kMGTPE"[exp]) -} diff --git a/config/env.go b/config/env.go index f482cd24733..06233c4b641 100644 --- a/config/env.go +++ b/config/env.go @@ -17,6 +17,9 @@ import ( "os" "runtime" "strconv" + + "github.com/pbnjay/memory" + "strings" ) @@ -33,6 +36,38 @@ func GetNumWorkerMultiplier() int { return runtime.NumCPU() } +const ( + gigabyte = 1 << 30 +) + +// GetMemoryLimit returns the upper memory limit in bytes for Hugo's in-memory caches. +// Note that this does not represent "all of the memory" that Hugo will use, +// so it needs to be set to a lower number than the available system memory. +// It will read from the HUGO_MEMORYLIMIT (in Gigabytes) environment variable. +// If that is not set, it will set aside a quarter of the total system memory. +func GetMemoryLimit() uint64 { + if mem := os.Getenv("HUGO_MEMORYLIMIT"); mem != "" { + if v := stringToGibabyte(mem); v > 0 { + return v + } + + } + + m := memory.TotalMemory() + if m != 0 { + return uint64(m / 4) + } + + return 2 * gigabyte +} + +func stringToGibabyte(f string) uint64 { + if v, err := strconv.ParseFloat(f, 32); err == nil && v > 0 { + return uint64(v * gigabyte) + } + return 0 +} + // SetEnvVars sets vars on the form key=value in the oldVars slice. func SetEnvVars(oldVars *[]string, keyValues ...string) { for i := 0; i < len(keyValues); i += 2 { diff --git a/config/env_test.go b/config/env_test.go index 3c402b9ef8d..3b0faac3b7d 100644 --- a/config/env_test.go +++ b/config/env_test.go @@ -29,4 +29,6 @@ func TestSetEnvVars(t *testing.T) { key, val := SplitEnvVar("HUGO=rocks") c.Assert(key, qt.Equals, "HUGO") c.Assert(val, qt.Equals, "rocks") + + c.Assert(stringToGibabyte("1.0"), qt.Equals, uint64(1073741824)) } diff --git a/deps/deps.go b/deps/deps.go index b70dcb5a022..10b5bd07dc7 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -5,6 +5,8 @@ import ( "sync/atomic" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/cache/filecache" @@ -63,9 +65,12 @@ type Deps struct { // The configuration to use Cfg config.Provider `json:"-"` - // The file cache to use. + // The file caches to use. FileCaches filecache.Caches + // The memory cache to use. + MemCache *memcache.Cache + // The translation func to use Translate func(translationID string, templateData interface{}) string `json:"-"` @@ -162,6 +167,13 @@ type ResourceProvider interface { Clone(deps *Deps) error } +// Stop stops all running caches etc. +func (d *Deps) Stop() { + if d.MemCache != nil { + d.MemCache.Stop() + } +} + func (d *Deps) Tmpl() tpl.TemplateHandler { return d.tmpl } @@ -240,11 +252,12 @@ func New(cfg DepsCfg) (*Deps, error) { if err != nil { return nil, errors.WithMessage(err, "failed to create file caches from configuration") } + memCache := memcache.New(memcache.Config{Running: cfg.Running}) errorHandler := &globalErrHandler{} buildState := &BuildState{} - resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) + resourceSpec, err := resources.NewSpec(ps, fileCaches, memCache, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -284,6 +297,7 @@ func New(cfg DepsCfg) (*Deps, error) { Language: cfg.Language, Site: cfg.Site, FileCaches: fileCaches, + MemCache: memCache, BuildStartListeners: &Listeners{}, BuildState: buildState, Running: cfg.Running, @@ -319,7 +333,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er // The resource cache is global so reuse. // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) + d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.MemCache, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index 392f71a66f9..b605d61d753 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -371,9 +371,18 @@ Set `titleCaseStyle` to specify the title style used by the [title](/functions/t ## Configuration Environment Variables +### Control Parallel Processing + HUGO_NUMWORKERMULTIPLIER : Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used. +### Control Memory Usage + +HUGO_MEMORYLIMIT {{< new-in "0.75.0" >}} +: A value in Gigabytes (e.g. "1.5") to control the upper memory limit for Hugo's memory use. If not set, it will set aside a quarter of the total system memory. +Note that this does not represent a hard promise that Hugo will not use more memory than configured, but we will try our best. You may need to tune this setting depending on your site and/or system. + + ## Configuration Lookup Order Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior: diff --git a/go.mod b/go.mod index 3fb9588888e..e7e3790fb8a 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/google/go-cmp v0.5.2 github.com/gorilla/websocket v1.4.2 github.com/jdkato/prose v1.2.0 + github.com/karlseguin/ccache/v2 v2.0.7-0.20200816131247-1189f7f993b5 github.com/kylelemons/godebug v1.1.0 github.com/kyokomi/emoji v2.2.4+incompatible github.com/magefile/mage v1.10.0 @@ -39,6 +40,7 @@ require ( github.com/nicksnyder/go-i18n/v2 v2.1.1 github.com/niklasfasching/go-org v1.3.2 github.com/olekukonko/tablewriter v0.0.4 + github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 github.com/pelletier/go-toml v1.8.1 github.com/pkg/errors v0.9.1 github.com/rogpeppe/go-internal v1.6.2 diff --git a/go.sum b/go.sum index 49784a74f7f..714715b0948 100644 --- a/go.sum +++ b/go.sum @@ -7,16 +7,11 @@ cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISt cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= @@ -58,6 +53,8 @@ github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkK github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U= github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= github.com/alecthomas/chroma v0.7.2-0.20200305040604-4f3623dce67a/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s= +github.com/alecthomas/chroma v0.8.0 h1:HS+HE97sgcqjQGu5uVr8jIE55Mmh5UeQ7kckAhHg2pY= +github.com/alecthomas/chroma v0.8.0/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= github.com/alecthomas/chroma v0.8.1 h1:ym20sbvyC6RXz45u4qDglcgr8E313oPROshcuCHqiEE= github.com/alecthomas/chroma v0.8.1/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo= @@ -83,7 +80,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.19.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.35.0 h1:Pxqn1MWNfBCNcX7jrXCCTfsKpg5ms2IMUMmmcGtYJuo= +github.com/aws/aws-sdk-go v1.27.1 h1:MXnqY6SlWySaZAqNnXThOvjRFdiiOuKtC6i7baFdNdU= +github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.35.0/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -91,7 +89,8 @@ github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bep/gitmap v1.1.2 h1:zk04w1qc1COTZPPYWDQHvns3y1afOsdRfraFQ3qI840= github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= -github.com/bep/golibsass v0.7.0 h1:/ocxgtPZ5rgp7FA+mktzyent+fAg82tJq4iMsTMBAtA= +github.com/bep/golibsass v0.6.0 h1:WqJ8XC0Ri2210omWKwVVeaston02XhhArblb0ly6d6Y= +github.com/bep/golibsass v0.6.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= github.com/bep/golibsass v0.7.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= @@ -107,12 +106,14 @@ github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= @@ -135,6 +136,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/evanw/esbuild v0.6.5 h1:jVUDkQKOX9srwt/mUlvIba0/jmH46+B5wfwh0pWW7BM= +github.com/evanw/esbuild v0.6.5/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/evanw/esbuild v0.7.18 h1:HNMBF6AbyXOhocM4X0WuEQdbfh+/c1URzN0TbihicAA= github.com/evanw/esbuild v0.7.18/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -152,6 +155,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getkin/kin-openapi v0.14.0 h1:hqwQL7kze/adt0wB+0UJR2nJm+gfUHqM0Gu4D8nByVc= +github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/getkin/kin-openapi v0.22.1 h1:ODA1olTp175o//NfHko/uCAAhwUSfm5P4+K52XvTg4w= github.com/getkin/kin-openapi v0.22.1/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -181,7 +186,6 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -193,6 +197,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93 h1:VvBteXw2zOXEgm0o3PgONTWf+bhUGsCaiNn3pbkU9LA= +github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -210,7 +216,6 @@ github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -222,7 +227,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -254,24 +260,33 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jdkato/prose v1.1.1 h1:r6CwY09U97IZNgNQEHoeCh2nvg2e8WCOGjPH/b7lowI= +github.com/jdkato/prose v1.1.1/go.mod h1:jkF0lkxaX5PFSlk9l4Gh9Y+T57TqUZziWT7uZbW5ADg= github.com/jdkato/prose v1.2.0 h1:t/R3H6xOrVuIgNevWiOSJf1kEoeF2VWlrN6w76Tkzow= github.com/jdkato/prose v1.2.0/go.mod h1:WC4YKHtBdAMgBdmfdqBmEuVbBD0U5c9HQ6l1U8Cq0ts= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karlseguin/ccache v1.0.1 h1:0gpC6z1qtv0cKmsi5Su5tTB6bJ2vm9bfOLACpDEB/Ro= +github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= +github.com/karlseguin/ccache/v2 v2.0.6 h1:jFCLz4bF4EPfuCcvESAgYNClkEb31LV3WzyOwLlFz7w= +github.com/karlseguin/ccache/v2 v2.0.6/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200816110752-839a17bedbbc h1:KDIvcaeFw0dIbbhniUBaePOn11We4blME6Z4hkIfoAk= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200816110752-839a17bedbbc/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200816131247-1189f7f993b5 h1:oiSLAVaELZi+gW0jic4sNiI9cRq0/QdkDllDU15/aeA= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200816131247-1189f7f993b5/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA= +github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -279,6 +294,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -286,10 +303,13 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/kyokomi/emoji v2.2.1+incompatible h1:uP/6J5y5U0XxPh6fv8YximpVD1uMrshXG78I1+uF5SA= +github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= github.com/kyokomi/emoji v2.2.4+incompatible h1:np0woGKwx9LiHAQmwZx79Oc0rHpNw3o+3evou4BEPv4= github.com/kyokomi/emoji v2.2.4+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= +github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= +github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -334,8 +354,13 @@ github.com/muesli/smartcrop v0.3.0/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78Rwc github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/nicksnyder/go-i18n v1.10.0 h1:5AzlPKvXBH4qBzmZ09Ua9Gipyruv6uApMcrNZdo96+Q= +github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/nicksnyder/go-i18n v1.10.1 h1:isfg77E/aCD7+0lD/D00ebR2MV5vgeQ276WYyDaCRQc= github.com/nicksnyder/go-i18n/v2 v2.1.1 h1:ATCOanRDlrfKVB4WHAdJnLEqZtDmKYsweqsOUYflnBU= github.com/nicksnyder/go-i18n/v2 v2.1.1/go.mod h1:d++QJC9ZVf7pa48qrsRWhMJ5pSHIPmS3OLqK1niyLxs= +github.com/niklasfasching/go-org v1.3.1 h1:Hpw/eAGnIFS+BQxoBj7aa7zHz3L6lliYkhJOSv6F/8I= +github.com/niklasfasching/go-org v1.3.1/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU= github.com/niklasfasching/go-org v1.3.2 h1:ZKTSd+GdJYkoZl1pBXLR/k7DRiRXnmB96TRiHmHdzwI= github.com/niklasfasching/go-org v1.3.2/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= @@ -348,8 +373,12 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 h1:MfIUBZ1bz7TgvQLVa/yPJZOGeKEgs6eTKUjz3zB4B+U= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= +github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -380,22 +409,26 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.5.1 h1:asQ0uD7BN9RU5Im41SEEZTwCi/zAXdMOLS3npYaos2g= +github.com/rogpeppe/go-internal v1.5.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2 h1:aIihoIOHCiLZHxyoNQ+ABL4NKhFTgKLBdMLyEAh98m0= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6 h1:tlXG832s5pa9x9Gs3Rp2rTvEqjiDEuETUOSfBEiTcns= github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sanity-io/litter v1.2.0 h1:DGJO0bxH/+C2EukzOSBmAlxmkhVMGqzvcx/rvySYw9M= +github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= github.com/sanity-io/litter v1.3.0 h1:5ZO+weUsqdSWMUng5JnpkW/Oz8iTXiIdeumhQr1sSjs= github.com/sanity-io/litter v1.3.0/go.mod h1:5Z71SvaYy5kcGtyglXOC9rrUi3c1E8CamFWjQsazTh0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shogo82148/go-shuffle v0.0.0-20180218125048-27e6095f230d/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -406,13 +439,16 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.7 h1:FfTH+vuMXOas8jmfb5/M7dzEYx7LpcLb7a0LPe34uOU= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/fsync v0.9.0 h1:f9CEt3DOB2mnHxZaftmEOFWjABEvKM/xpf3cUwJrGOY= github.com/spf13/fsync v0.9.0/go.mod h1:fNtJEfG3HiltN3y4cPOz6MLjos9+2pIEqLIgszqhp/0= @@ -424,7 +460,10 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= +github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= @@ -438,7 +477,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -454,15 +492,20 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.22 h1:0e0f6Zee9SAQ5yOZGNMWaOxqVvcc/9/kUWu/Kl91Jk8= github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.31 h1:nKIhaVknZ0wOBBg0Uu6px+t218SfkLh2i/JwwOXYXqs= +github.com/yuin/goldmark v1.1.31/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 h1:VWSxtAiQNh3zgHJpdpkpVYjTPqRE3P6UZCOPa1nRDio= @@ -486,6 +529,7 @@ gocloud.dev v0.15.0/go.mod h1:ShXCyJaGrJu9y/7a6+DSCyBb9MFGZ1P5wwPa0Wu6w34= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -495,7 +539,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -507,7 +550,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -536,6 +578,8 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgP golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -543,7 +587,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFM golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0 h1:xFEXbcD0oa/xhqQmMXztdZ0bWvexAWds+8c1gRN8nu0= +golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -566,6 +611,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -611,9 +657,9 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -625,13 +671,13 @@ google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= +google.golang.org/appengine v1.6.0 h1:Tfd7cKwKbFRsI8RMAD3oqqw7JPFRrvFlOsfbgVkjOOw= +google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= @@ -641,10 +687,11 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 h1:4rNOqY4ULrKzS6twXa619uQgI7h9PaVd4ZhjFQ7C5zs= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= @@ -653,7 +700,6 @@ google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -680,13 +726,11 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= pack.ag/amqp v0.8.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= diff --git a/helpers/general.go b/helpers/general.go index 80e303087a5..377cba65578 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -61,6 +61,24 @@ func FindAvailablePort() (*net.TCPAddr, error) { return nil, err } +// FormatByteCount pretty formats b. +func FormatByteCount(bc uint64) string { + const ( + Gigabyte = 1 << 30 + Megabyte = 1 << 20 + Kilobyte = 1 << 10 + ) + switch { + case bc > Gigabyte || -bc > Gigabyte: + return fmt.Sprintf("%.2f GB", float64(bc)/Gigabyte) + case bc > Megabyte || -bc > Megabyte: + return fmt.Sprintf("%.2f MB", float64(bc)/Megabyte) + case bc > Kilobyte || -bc > Kilobyte: + return fmt.Sprintf("%.2f KB", float64(bc)/Kilobyte) + } + return fmt.Sprintf("%d B", bc) +} + // InStringArray checks if a string is an element of a slice of strings // and returns a boolean value. func InStringArray(arr []string, el string) bool { diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 79d89a88b56..9bd44185712 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -40,6 +40,7 @@ const ( metaKeyBaseDir = "baseDir" // Abs base directory of source file. metaKeyMountRoot = "mountRoot" metaKeyModule = "module" + metaKeyComponent = "component" metaKeyOriginalFilename = "originalFilename" metaKeyName = "name" metaKeyPath = "path" @@ -132,6 +133,10 @@ func (f FileMeta) MountRoot() string { return f.stringV(metaKeyMountRoot) } +func (f FileMeta) Component() string { + return f.stringV(metaKeyComponent) +} + func (f FileMeta) Module() string { return f.stringV(metaKeyModule) } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 2c4f0df52e4..cd430a79faf 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -63,6 +63,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rm.Meta[metaKeyBaseDir] = rm.ToBasedir rm.Meta[metaKeyMountRoot] = rm.path rm.Meta[metaKeyModule] = rm.Module + rm.Meta[metaKeyComponent] = fromBase meta := copyFileMeta(rm.Meta) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index e79228ba340..70840b2d46e 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -251,6 +251,13 @@ func (m *pageMap) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resour } target := strings.TrimPrefix(meta.Path(), owner.File().Dir()) + tbase := meta.TranslationBaseNameWithExt() + if tbase != "" { + dir, base := filepath.Split(target) + if base != tbase { + target = filepath.Join(dir, tbase) + } + } return owner.s.ResourceSpec.New( resources.ResourceSourceDescriptor{ diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 189aa19c6cc..c694bd81e5a 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -20,6 +20,7 @@ import ( "os" "path" "path/filepath" + "sort" "strings" "sync" @@ -258,6 +259,19 @@ func (s SourceFilesystems) IsAsset(filename string) bool { return s.Assets.Contains(filename) } +// CollectResourcePaths collects paths relative to their component root from +// the data, content and assets filesystems. +func (s SourceFilesystems) CollectResourcePaths(filename string) []string { + var paths []string + + for _, fs := range []*SourceFilesystem{s.Assets, s.Content, s.Data} { + paths = append(paths, fs.collectRelativePaths(filename)...) + } + + return uniqueStringsSorted(paths) + +} + // IsI18n returns true if the given filename is a member of the i18n filesystem. func (s SourceFilesystems) IsI18n(filename string) bool { return s.I18n.Contains(filename) @@ -277,21 +291,40 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { // MakePathRelative creates a relative path from the given filename. // It will return an empty string if the filename is not a member of this filesystem. +// TODO(bep) there may be multiple, see https://github.com/gohugoio/hugo/issues/7563 +// See collectRelativePaths func (d *SourceFilesystem) MakePathRelative(filename string) string { + paths := d.collectRelativePaths(filename) + if paths == nil { + return "" + } + return paths[0] + +} +func (d *SourceFilesystem) collectRelativePaths(filename string) []string { + var paths []string for _, dir := range d.Dirs { meta := dir.(hugofs.FileMetaInfo).Meta() currentPath := meta.Filename() - - if strings.HasPrefix(filename, currentPath) { - rel := strings.TrimPrefix(filename, currentPath) + if rel := relFilename(currentPath, filename); rel != "" { if mp := meta.Path(); mp != "" { rel = filepath.Join(mp, rel) } - return strings.TrimPrefix(rel, filePathSeparator) + paths = append(paths, rel) } } - return "" + return paths +} + +func relFilename(dirname, filename string) string { + if !strings.HasSuffix(dirname, filePathSeparator) { + dirname += filePathSeparator + } + if !strings.HasPrefix(filename, dirname) { + return "" + } + return strings.TrimPrefix(filename, dirname) } func (d *SourceFilesystem) RealFilename(rel string) string { @@ -770,3 +803,21 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollect return b.createOverlayFs(collector, mounts[1:]) } + +func uniqueStringsSorted(s []string) []string { + if len(s) == 0 { + return nil + } + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 3c0440a9768..b74f813d9d2 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -51,6 +51,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Make sure we don't trigger rebuilds in parallel. h.runningMu.Lock() defer h.runningMu.Unlock() + } else { + defer func() { + h.Stop() + }() } ctx, task := trace.NewTask(context.Background(), "Build") diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 8d0872bd5e0..88ac32fa10b 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -2,6 +2,7 @@ package hugolib import ( "fmt" + "os" "strings" "testing" @@ -11,6 +12,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource_transformers/tocss/scss" "github.com/fortytw2/leaktest" "github.com/fsnotify/fsnotify" @@ -417,15 +419,18 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { func TestMultiSitesRebuild(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 - // This leaktest seems to be a little bit shaky on Travis. - if !isCI() { - defer leaktest.CheckTimeout(t, 10*time.Second)() - } c := qt.New(t) b := newMultiSiteTestDefaultBuilder(t).Running().CreateSites().Build(BuildCfg{}) + defer b.H.Stop() + + // This leaktest seems to be a little bit shaky on Travis. + if !isCI() { + defer leaktest.CheckTimeout(t, 10*time.Second)() + } + sites := b.H.Sites fs := b.Fs @@ -1478,3 +1483,71 @@ func TestRebuildOnAssetChange(t *testing.T) { b.Build(BuildCfg{}) b.AssertFileContent("public/index.html", `changed data`) } + +func TestCacheOnRebuild(t *testing.T) { + c := qt.New(t) + + createSitesBuilder := func(t testing.TB, prepare func(b *sitesBuilder)) *sitesBuilder { + b := newTestSitesBuilder(t).Running() + prepare(b) + b.Build(BuildCfg{}) + return b + + } + + c.Run("transform.Unmarshal with temporary Resource", func(c *qt.C) { + templ := ` + {{ $m := "a: 1" | resources.FromString "a.yaml" | transform.Unmarshal }} + NM: {{ $m }} + + ` + b := createSitesBuilder(c, func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", templ) + }) + + b.AssertFileContent("public/index.html", `NM: map[a:1]`) + + templ = strings.ReplaceAll(templ, "a: 1", "a: 2") + templ = strings.ReplaceAll(templ, "NM:", "NM2:") + b.EditFiles("layouts/index.html", templ) + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `NM2: map[a:2]`) + + }) + + c.Run("SCSS with multiple roots", func(c *qt.C) { + if !scss.Supports() { + c.Skip("Skip SCSS") + } + + b, workDir, clean := newTestSitesBuilderWithOSFs(c, "scss-multiroot") + defer clean() + b.Running() + + assetsDir := filepath.Join(workDir, "assets") + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root1"), 0777), qt.IsNil) + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root2"), 0777), qt.IsNil) + + b.WithTemplatesAdded("index.html", ` +{{ $css := resources.Get "root2/main.scss" | toCSS }} +CSS: {{ $css.Content | safeHTML }} +`) + + b.WithSourceFile(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: blue;`) + b.WithSourceFile(filepath.Join(assetsDir, "root2/main.scss"), `@import "../root1/_vars"; + +body { color: $base-color; } +`) + + b.Build(BuildCfg{}) + b.AssertFileContent("public/index.html", `color: blue;`) + + b.EditFiles(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: red;`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `color: red;`) + }) + +} diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 4fe4960422c..80885b5d67c 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -89,7 +89,7 @@ languageName = "Nynorsk" s2h := s2.getPage(page.KindHome) c.Assert(s2h.Permalink(), qt.Equals, "https://example.fr/") - b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /docs/text/pipes.txt") + b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /text/pipes.txt") b.AssertFileContent("public/fr/text/pipes.txt", "Hugo Pipes") b.AssertFileContent("public/en/index.html", "Default Home Page", "String Resource: /docs/text/pipes.txt") b.AssertFileContent("public/en/text/pipes.txt", "Hugo Pipes") diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index fa420a025aa..5af55e738f0 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -347,7 +347,7 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { b.AssertFileContent("public/en/bc/data1.json", "data1") b.AssertFileContent("public/en/bc/data2.json", "data2") b.AssertFileContent("public/en/bc/logo-bc.png", "logo") - b.AssertFileContent("public/nn/bc/data1.nn.json", "data1.nn") + b.AssertFileContent("public/nn/bc/data1.json", "data1.nn") b.AssertFileContent("public/nn/bc/data2.json", "data2") b.AssertFileContent("public/nn/bc/logo-bc.png", "logo") @@ -1320,7 +1320,7 @@ bundle min min key: {{ $jsonMinMin.Key }} b.AssertFileContent(index, fmt.Sprintf("bundle min min min: /bundle%d/data.min.min.min.json", i), - fmt.Sprintf("bundle min min key: /bundle%d/data.min.min.json", i), + fmt.Sprintf("bundle min min key: bundle%d/content/en/data_c9a5f8c93697691d6f8a298931d285f7.json", i), ) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.min.min.json", i)), qt.Equals, true) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.json", i)), qt.Equals, false) diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 9f2e4bab137..346f3dd2d64 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -184,10 +184,6 @@ func (c *pagesCollector) Collect() (collectErr error) { } for dir := range dirs { - for _, pm := range c.contentMap.pmaps { - pm.s.ResourceSpec.DeleteBySubstring(dir.dirname) - } - switch dir.tp { case bundleLeaf: collectErr = c.collectDir(dir.dirname, true, nil) diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 97d4f17bab7..066c788fd16 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -53,6 +53,8 @@ type Paths struct { PublishDir string + IsMultiHost bool + // When in multihost mode, this returns a list of base paths below PublishDir // for each language. MultihostTargetBasePaths []string @@ -142,7 +144,8 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { } var multihostTargetBasePaths []string - if languages.IsMultihost() { + isMultiHost := languages.IsMultihost() + if isMultiHost { for _, l := range languages { multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang) } @@ -171,6 +174,7 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { Language: language, Languages: languages, LanguagesDefaultFirst: languagesDefaultFirst, + IsMultiHost: isMultiHost, MultihostTargetBasePaths: multihostTargetBasePaths, PaginatePath: cfg.GetString("paginatePath"), diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 7573199aacd..827bc04f5ae 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -355,7 +355,7 @@ Edited content. `) b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil) - b.H.ResourceSpec.ClearCaches() + b.H.MemCache.Clear() } } diff --git a/hugolib/resource_change_test.go b/hugolib/resource_change_test.go new file mode 100644 index 00000000000..941df8e7ea0 --- /dev/null +++ b/hugolib/resource_change_test.go @@ -0,0 +1,322 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "strings" + "testing" +) + +func TestResourceEditMetadata(t *testing.T) { + b := newTestSitesBuilder(t).Running() + + content := `+++ +title = "My Bundle With TOML Meta" + +[[resources]] +src = "**.toml" +title = "My TOML :counter" ++++ + +Content. +` + + b.WithContent( + "bundle/index.md", content, + "bundle/my1.toml", `a = 1`, + "bundle/my2.toml", `a = 2`) + + b.WithTemplatesAdded("index.html", ` +{{ $bundle := site.GetPage "bundle" }} +{{ $toml := $bundle.Resources.GetMatch "*.toml" }} +TOML: {{ $toml.Title }} + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My TOML 1") + + b.EditFiles("content/bundle/index.md", strings.ReplaceAll(content, "My TOML", "My Changed TOML 1")) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My Changed TOML") + +} + +func TestResourceCacheSimpleTest(t *testing.T) { + conf := ` +baseURL = "https://example.org" + +defaultContentLanguage = "en" + +[module] +[[module.mounts]] +source = "content/cen" +target = "content" +lang="en" +[[module.mounts]] +source = "content/cno" +target = "content" +lang="no" +[[module.mounts]] +source = "assets" +target = "assets" +[[module.mounts]] +source = "assets_common" +target = "assets/aen" +[[module.mounts]] +source = "assets_common" +target = "assets/ano" + +[languages] +[languages.en] +weight = 1 + +[languages.no] +weight = 2 + +` + b := newTestSitesBuilder(t).WithConfigFile("toml", conf).Running() + + b.WithSourceFile( + "content/cen/bundle/index.md", "---\ntitle: En Bundle\n---", + "content/cen/bundle/data1.json", `{ "data1": "en" }`, + "content/cen/bundle/data2.json", `{ "data2": "en" }`, + "content/cno/bundle/index.md", "---\ntitle: No Bundle\n---", + "content/cno/bundle/data1.json", `{ "data1": "no" }`, + "content/cno/bundle/data3.json", `{ "data3": "no" }`, + ) + + b.WithSourceFile("assets_common/data/common.json", `{ + "Hugo": "Rocks!", + }`) + + b.WithSourceFile("assets/data/mydata.json", `{ + "a": 32, + }`) + + b.WithTemplatesAdded("index.html", ` +{{ $data := resources.Get "data/mydata.json" }} +{{ template "print-resource" ( dict "title" "data" "r" $data ) }} +{{ $dataMinified := $data | minify }} +{{ template "print-resource" ( dict "title" "data-minified" "r" $dataMinified ) }} +{{ $dataUnmarshaled := $dataMinified | transform.Unmarshal }} +Data Unmarshaled: {{ $dataUnmarshaled }} +{{ $bundle := site.GetPage "bundle" }} +{{ range (seq 3) }} +{{ $i := . }} +{{ with $bundle.Resources.GetMatch (printf "data%d.json" . ) }} +{{ $minified := . | minify }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d" $i) "r" . ) }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d min" $i) "r" $minified ) }} +{{ end }} +{{ end }} +{{ $common1 := resources.Get "aen/data/common.json" }} +{{ $common2 := resources.Get "ano/data/common.json" }} +{{ template "print-resource" ( dict "title" "common1" "r" $common1 ) }} +{{ template "print-resource" ( dict "title" "common2" "r" $common2 ) }} +{{ define "print-resource" }}{{ .title }}|{{ .r.RelPermalink }}|{{ .r.Key }}|{{ .r.Content | safeHTML }}|{{ end }} + + + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +Data Unmarshaled: map[a:32] +bundle data 1|/bundle/data1.json|bundle/content/en/data1.json|{ "data1": "en" }| +bundle data 1 min|/bundle/data1.min.json|bundle/content/en/data1_d3f53f09220d597dac26fe7840c31fc9.json|{"data1":"en"}| +bundle data 3|/bundle/data3.json|bundle/content/en/data3.json|{ "data3": "no" }| +bundle data 3 min|/bundle/data3.min.json|bundle/content/en/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json| +common2|/ano/data/common.json|ano/data/common.json| +`) + + b.AssertFileContent("public/no/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +bundle data 1|/no/bundle/data1.json|bundle/content/no/data1.json|{ "data1": "no" }| +bundle data 2|/no/bundle/data2.json|bundle/content/no/data2.json|{ "data2": "en" }| + bundle data 3|/no/bundle/data3.json|bundle/content/no/data3.json|{ "data3": "no" }| +bundle data 3 min|/no/bundle/data3.min.json|bundle/content/no/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json +`) + + b.EditFiles("assets/data/mydata.json", `{ "a": 42 }`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data|/data/mydata.json|data/mydata.json|{ "a": 42 }| +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":42}| +Data Unmarshaled: map[a:42] +`) + +} + +func TestResourceCacheMultihost(t *testing.T) { + toLang := func(format, lang string) string { + return fmt.Sprintf(format, lang) + } + + addContent := func(b *sitesBuilder, contentPath func(path, lang string) string) { + b.WithNoContentAdded() + for _, lang := range []string{"en", "fr"} { + b.WithSourceFile( + contentPath("b1/index.md", lang), toLang("---\ntitle: Bundle 1 %s\n---", lang), + contentPath("b1/styles/style11.css", lang), toLang(".%s1: { color: blue };", lang), + contentPath("b1/styles/style12.css", lang), toLang(".%s2: { color: red };", lang), + ) + b.WithSourceFile( + contentPath("b2/index.md", lang), toLang("---\ntitle: Bundle 2 %s\n---", lang), + contentPath("b2/styles/style21.css", lang), toLang(".%s21: { color: green };", lang), + contentPath("b2/styles/style22.css", lang), toLang(".%s22: { color: orange };", lang), + ) + } + } + + addTemplates := func(b *sitesBuilder) { + b.WithTemplates("_default/single.html", ` +{{ template "print-page" (dict "page" . "title" "Self") }} +{{ $other := site.Sites.First.GetPage "b1" }} +{{ template "print-page" (dict "page" $other "title" "Other") }} + + +{{ define "print-page" }} +{{ $p := .page }} +{{ $title := .title }} +{{ $styles := $p.Resources.Match "**/style1*.css" }} +{{ if $styles }} +{{ $firststyle := index $styles 0 }} +{{ $mystyles := $styles | resources.Concat "mystyles.css" }} +{{ $title }} Mystyles First CSS: {{ $firststyle.RelPermalink }}|Key: {{ $firststyle.Key }}|{{ $firststyle.Content }}| +{{ $title }} Mystyles CSS: {{ $mystyles.RelPermalink }}|Key: {{ $mystyles.Key }}|{{ $mystyles.Content }}| +{{ end }} +{{ $title }} Bundle: {{ $p.Permalink }} +{{ $style := $p.Resources.GetMatch "**.css" }} +{{ $title }} CSS: {{ $style.RelPermalink }}|Key: {{ $style.Key }}|{{ $style.Content }}| +{{ $minified := $style | minify }} +{{ $title }} Minified CSS: {{ $minified.RelPermalink }}|Key: {{ $minified.Key }}|{{ $minified.Content }} +{{ end }} + +`) + } + + assertContent := func(b *sitesBuilder) { + + otherAssert := `Other Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Other Bundle: https://example.com/b1/ +Other CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue }; +Other Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue}` + + b.AssertFileContent("public/fr/b1/index.html", ` +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red }; +Self Bundle: https://example.fr/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/fr/styles/style11.css|.fr1: { color: blue }; +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/fr/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.fr1:{color:blue} +`, + otherAssert) + b.AssertFileContent("public/en/b1/index.html", ` +Self Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red };| +Self Bundle: https://example.com/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue} +`, otherAssert) + + b.AssertFileContent("public/fr/b2/index.html", ` +Self Bundle: https://example.fr/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/fr/styles/style21.css|.fr21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/fr/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.fr21:{color:green} +`, otherAssert) + + b.AssertFileContent("public/en/b2/index.html", ` +Self Bundle: https://example.com/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/en/styles/style21.css|.en21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/en/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.en21:{color:green} +`, otherAssert) + + } + + t.Run("Default content", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false +contentDir = "content" + +[Languages] +[Languages.en] +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + fmt.Println(b.workingDir) + addContent(b, func(path, lang string) string { + path = strings.Replace(path, ".", "."+lang+".", 1) + path = "content/" + path + return path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + + t.Run("Content dir per language", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false + +[Languages] +[Languages.en] +contentDir = "content_en" +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +contentDir = "content_fr" +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + addContent(b, func(path, lang string) string { + return "content_" + lang + "/" + path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + +} diff --git a/hugolib/site.go b/hugolib/site.go index ec293953002..ea4c415380f 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -23,18 +23,17 @@ import ( "os" "path" "path/filepath" - "regexp" "sort" "strconv" "strings" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/constants" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/converter/hooks" @@ -1028,20 +1027,20 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro logger = helpers.NewDistinctFeedbackLogger() ) - var isCSSConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`) - var isCSSFileRe = regexp.MustCompile(`\.(css|scss|sass)`) - - var cachePartitions []string + // TODO1 + //var isCSSConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`) + //var isCSSFileRe = regexp.MustCompile(`\.(css|scss|sass)`) + //var cachePartitions []string // Special case // TODO(bep) I have a ongoing branch where I have redone the cache. Consider this there. - var isCSSChange bool + //var isCSSChange bool + + // Paths relative to their component folder in data, assets or content. + var resourcePaths []string for _, ev := range events { - if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { - cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) - if !isCSSChange { - isCSSChange = isCSSFileRe.MatchString(assetsFilename) || isCSSConfigRe.MatchString(assetsFilename) - } + if paths := s.BaseFs.CollectResourcePaths(ev.Name); paths != nil { + resourcePaths = append(resourcePaths, paths...) } id, found := s.eventToIdentity(ev) @@ -1085,13 +1084,16 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro return err } + // TODO1 // These in memory resource caches will be rebuilt on demand. - for _, s := range s.h.Sites { - s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) - if isCSSChange { - s.ResourceSpec.ResourceCache.DeleteContains("css", "scss", "sass") - } - } + //for _, s := range s.h.Sites { + // s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) + // if isCSSChange { + // s.ResourceSpec.ResourceCache.DeleteContains("css", "scss", "sass") + // } + //} + + h.MemCache.ClearOn(memcache.ClearOnRebuild, resourcePaths...) if tmplChanged || i18nChanged { sites := s.h.Sites diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 5b825cd1e1c..bc6df1c3da1 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -92,7 +92,8 @@ type sitesBuilder struct { // Consider this in relation to using the BaseFs.PublishFs to all publishing. workingDir string - addNothing bool + addNothing bool + addNoContent bool // Base data/content contentFilePairs []filenameContent templateFilePairs []filenameContent @@ -126,6 +127,21 @@ func newTestSitesBuilder(t testing.TB) *sitesBuilder { dumper: litterOptions, rnd: rand.New(rand.NewSource(time.Now().Unix()))} } +func newTestSitesBuilderWithOSFs(t testing.TB, testname string) (*sitesBuilder, string, func()) { + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, testname) + if err != nil { + t.Fatal(err) + } + v := viper.New() + v.Set("workingDir", workDir) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) + + return b, workDir, clean +} + func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder { c := qt.New(t) @@ -154,6 +170,11 @@ func (s *sitesBuilder) WithNothingAdded() *sitesBuilder { return s } +func (s *sitesBuilder) WithNoContentAdded() *sitesBuilder { + s.addNoContent = true + return s +} + func (s *sitesBuilder) WithLogger(logger loggers.Logger) *sitesBuilder { s.logger = logger return s @@ -502,6 +523,7 @@ func (s *sitesBuilder) LoadConfig() error { } func (s *sitesBuilder) CreateSitesE() error { + if !s.addNothing { if _, ok := s.Fs.Source.(*afero.OsFs); ok { for _, dir := range []string{ @@ -529,7 +551,6 @@ func (s *sitesBuilder) CreateSitesE() error { s.writeFilePairs("data", s.dataFilePairs) s.writeFilePairs("content", s.contentFilePairs) s.writeFilePairs("layouts", s.templateFilePairs) - } if err := s.LoadConfig(); err != nil { @@ -677,7 +698,7 @@ hello: } ) - if len(s.contentFilePairs) == 0 { + if len(s.contentFilePairs) == 0 && !s.addNoContent { s.writeFilePairs("content", s.createFilenameContent(defaultContent)) } diff --git a/modules/client.go b/modules/client.go index d07483d36a2..7aa5853cf93 100644 --- a/modules/client.go +++ b/modules/client.go @@ -355,7 +355,7 @@ var verifyErrorDirRe = regexp.MustCompile(`dir has been modified \((.*?)\)`) // which are stored in a local downloaded source cache, have not been // modified since being downloaded. func (c *Client) Verify(clean bool) error { - // TODO1 add path to mod clean + // TODO(bep) add path to mod clean err := c.runVerify() if err != nil { diff --git a/resources/image.go b/resources/image.go index e999c5d96c2..78f390b4ec1 100644 --- a/resources/image.go +++ b/resources/image.go @@ -371,7 +371,7 @@ func (i *imageResource) getImageMetaCacheTargetPath() string { df.dir = filepath.Dir(fi.Meta().Path()) } p1, _ := helpers.FileAndExt(df.file) - h, _ := i.hash() + h := i.hash() idStr := helpers.HashString(h, i.size(), imageMetaVersionNumber, cfg) return path.Join(df.dir, fmt.Sprintf("%s_%s.json", p1, idStr)) } @@ -382,7 +382,7 @@ func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile p2 = conf.TargetFormat.DefaultExtension() } - h, _ := i.hash() + h := i.hash() idStr := fmt.Sprintf("_hu%s_%d", h, i.size()) // Do not change for no good reason. diff --git a/resources/image_cache.go b/resources/image_cache.go index 1888b457f59..b1db71cbea6 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -17,8 +17,8 @@ import ( "image" "io" "path/filepath" - "strings" - "sync" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/images" @@ -30,36 +30,7 @@ type imageCache struct { pathSpec *helpers.PathSpec fileCache *filecache.Cache - - mu sync.RWMutex - store map[string]*resourceAdapter -} - -func (c *imageCache) deleteIfContains(s string) { - c.mu.Lock() - defer c.mu.Unlock() - s = c.normalizeKeyBase(s) - for k := range c.store { - if strings.Contains(k, s) { - delete(c.store, k) - } - } -} - -// The cache key is a lowecase path with Unix style slashes and it always starts with -// a leading slash. -func (c *imageCache) normalizeKey(key string) string { - return "/" + c.normalizeKeyBase(key) -} - -func (c *imageCache) normalizeKeyBase(key string) string { - return strings.Trim(strings.ToLower(filepath.ToSlash(key)), "/") -} - -func (c *imageCache) clear() { - c.mu.Lock() - defer c.mu.Unlock() - c.store = make(map[string]*resourceAdapter) + mCache memcache.Getter } func (c *imageCache) getOrCreate( @@ -67,101 +38,92 @@ func (c *imageCache) getOrCreate( createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) { relTarget := parent.relTargetPathFromConfig(conf) memKey := parent.relTargetPathForRel(relTarget.path(), false, false, false) - memKey = c.normalizeKey(memKey) + memKey = memcache.CleanKey(memKey) - // For the file cache we want to generate and store it once if possible. - fileKeyPath := relTarget - if fi := parent.root.getFileInfo(); fi != nil { - fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) - } - fileKey := fileKeyPath.path() + v, err := c.mCache.GetOrCreate(memKey, func() memcache.Entry { + // For the file cache we want to generate and store it once if possible. + fileKeyPath := relTarget + if fi := parent.root.getFileInfo(); fi != nil { + fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) + } + fileKey := fileKeyPath.path() - // First check the in-memory store, then the disk. - c.mu.RLock() - cachedImage, found := c.store[memKey] - c.mu.RUnlock() + var img *imageResource - if found { - return cachedImage, nil - } + // These funcs are protected by a named lock. + // read clones the parent to its new name and copies + // the content to the destinations. + read := func(info filecache.ItemInfo, r io.ReadSeeker) error { + img = parent.clone(nil) + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - var img *imageResource + if err := img.InitConfig(r); err != nil { + return err + } - // These funcs are protected by a named lock. - // read clones the parent to its new name and copies - // the content to the destinations. - read := func(info filecache.ItemInfo, r io.ReadSeeker) error { - img = parent.clone(nil) - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) + r.Seek(0, 0) - if err := img.InitConfig(r); err != nil { - return err - } + w, err := img.openDestinationsForWriting() + if err != nil { + return err + } - r.Seek(0, 0) + if w == nil { + // Nothing to write. + return nil + } + + defer w.Close() + _, err = io.Copy(w, r) - w, err := img.openDestinationsForWriting() - if err != nil { return err } - if w == nil { - // Nothing to write. - return nil - } + // create creates the image and encodes it to the cache (w). + create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + defer w.Close() - defer w.Close() - _, err = io.Copy(w, r) + var conv image.Image + img, conv, err = createImage() + if err != nil { + return + } + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - return err - } + return img.EncodeTo(conf, conv, w) + } + + // Now look in the file cache. - // create creates the image and encodes it to the cache (w). - create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { - defer w.Close() + // The definition of this counter is not that we have processed that amount + // (e.g. resized etc.), it can be fetched from file cache, + // but the count of processed image variations for this site. + c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - var conv image.Image - img, conv, err = createImage() + _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { - return + return memcache.Entry{Err: err} } - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) - return img.EncodeTo(conf, conv, w) - } + // The file is now stored in this cache. + img.setSourceFs(c.fileCache.Fs) - // Now look in the file cache. + imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - // The definition of this counter is not that we have processed that amount - // (e.g. resized etc.), it can be fetched from file cache, - // but the count of processed image variations for this site. - c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) + return memcache.Entry{Value: imgAdapter} + }) - _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { return nil, err } + return v.(*resourceAdapter), nil - // The file is now stored in this cache. - img.setSourceFs(c.fileCache.Fs) - - c.mu.Lock() - if cachedImage, found = c.store[memKey]; found { - c.mu.Unlock() - return cachedImage, nil - } - - imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - c.store[memKey] = imgAdapter - c.mu.Unlock() - - return imgAdapter, nil } -func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { - return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)} +func newImageCache(fileCache *filecache.Cache, memCache *memcache.Cache, ps *helpers.PathSpec) *imageCache { + return &imageCache{fileCache: fileCache, mCache: memCache.GetOrCreatePartition("images", memcache.ClearOnChange), pathSpec: ps} } diff --git a/resources/image_test.go b/resources/image_test.go index 1be9a5f8d0e..2d3020a6241 100644 --- a/resources/image_test.go +++ b/resources/image_test.go @@ -355,7 +355,7 @@ func TestImageResizeInSubPath(t *testing.T) { c.Assert(spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) // Cleare mem cache to simulate reading from the file cache. - spec.imageCache.clear() + spec.imageCache.mCache.Clear() resizedAgain, err := image.Resize("101x101") c.Assert(err, qt.IsNil) diff --git a/resources/resource.go b/resources/resource.go index acdf2d744ec..5209c0a9796 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -22,6 +22,8 @@ import ( "path/filepath" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/resources/internal" "github.com/gohugoio/hugo/common/herrors" @@ -172,8 +174,6 @@ func (commonResource) Slice(in interface{}) (interface{}, error) { return nil, fmt.Errorf("type %T is not a Resource", v) } groups[i] = g - { - } } return groups, nil default: @@ -197,8 +197,24 @@ type fileInfo interface { setSourceFilename(string) setSourceFs(afero.Fs) getFileInfo() hugofs.FileMetaInfo - hash() (string, error) size() int + hashProvider +} + +type hashProvider interface { + hash() string +} + +type resourceOrigin struct { + stale bool +} + +func (s *resourceOrigin) MarkStale() { + s.stale = true +} + +func (s *resourceOrigin) IsStale() bool { + return s.stale } // genericResource represents a generic linkable resource. @@ -235,7 +251,25 @@ func (l *genericResource) Data() interface{} { } func (l *genericResource) Key() string { - return l.RelPermalink() + // TODO1 consider repating the section in the path segment. + + if l.fi != nil { + // Create a key that at least shares the base folder with the source, + // to facilitate effective cache busting on changes. + meta := l.fi.Meta() + p := meta.Path() + if p != "" { + d, _ := filepath.Split(p) + p = path.Join(d, l.relTargetDirFile.file) + key := memcache.CleanKey(p) + key = memcache.InsertKeyPathElements(key, meta.Component(), meta.Lang()) + + return key + } + } + key := memcache.CleanKey(l.RelPermalink()) + + return key } func (l *genericResource) MediaType() media.Type { @@ -417,6 +451,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour } fpath, fname := path.Split(u.targetPath) + r.resourcePathDescriptor.relTargetDirFile = dirFile{dir: fpath, file: fname} r.mergeData(u.data) @@ -623,26 +658,25 @@ func (fi *resourceFileInfo) setSourceFs(fs afero.Fs) { fi.sourceFs = fs } -func (fi *resourceFileInfo) hash() (string, error) { +func (fi *resourceFileInfo) hash() string { var err error fi.h.init.Do(func() { var hash string var f hugio.ReadSeekCloser f, err = fi.ReadSeekCloser() if err != nil { - err = errors.Wrap(err, "failed to open source file") - return + panic(fmt.Sprintf("failed to open source file: %s", err)) } defer f.Close() hash, err = helpers.MD5FromFileFast(f) if err != nil { - return + panic(err) } fi.h.value = hash }) - return fi.h.value, err + return fi.h.value } func (fi *resourceFileInfo) size() int { diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index f42372fa396..30ecaeec3a4 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -179,6 +179,36 @@ type UnmarshableResource interface { Identifier } +// Staler controls stale state of a Resource. A stale resource should be discarded. +type Staler interface { + MarkStale() + StaleInfo +} + +// StaleInfo tells if a resource is marked as stale. +type StaleInfo interface { + IsStale() bool +} + +// IsStaleAny reports whether any of the os is marked as stale. +func IsStaleAny(os ...interface{}) bool { + for _, o := range os { + if s, ok := o.(StaleInfo); ok && s.IsStale() { + return true + } + } + return false +} + +// MarkStale will mark any of the oses as stale, if possible. +func MarkStale(os ...interface{}) { + for _, o := range os { + if s, ok := o.(Staler); ok { + s.MarkStale() + } + } +} + type resourceTypesHolder struct { mediaType media.Type resourceType string diff --git a/resources/resource_cache.go b/resources/resource_cache.go index feaa94f5cf0..eccc95e022e 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -16,25 +16,13 @@ package resources import ( "encoding/json" "io" - "path" - "path/filepath" - "strings" "sync" - "github.com/gohugoio/hugo/helpers" - - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/cache/filecache" - - "github.com/BurntSushi/locker" -) - -const ( - CACHE_CLEAR_ALL = "clear_all" - CACHE_OTHER = "other" ) type ResourceCache struct { @@ -42,123 +30,30 @@ type ResourceCache struct { sync.RWMutex - // Either resource.Resource or resource.Resources. - cache map[string]interface{} + // Memory cache with either + // resource.Resource or resource.Resources. + cache memcache.Getter fileCache *filecache.Cache - - // Provides named resource locks. - nlocker *locker.Locker -} - -// ResourceCacheKey converts the filename into the format used in the resource -// cache. -func ResourceCacheKey(filename string) string { - filename = filepath.ToSlash(filename) - return path.Join(resourceKeyPartition(filename), filename) -} - -func resourceKeyPartition(filename string) string { - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") - if ext == "" { - ext = CACHE_OTHER - } - return ext -} - -// Commonly used aliases and directory names used for some types. -var extAliasKeywords = map[string][]string{ - "sass": []string{"scss"}, - "scss": []string{"sass"}, -} - -// ResourceKeyPartitions resolves a ordered slice of partitions that is -// used to do resource cache invalidations. -// -// We use the first directory path element and the extension, so: -// a/b.json => "a", "json" -// b.json => "json" -// -// For some of the extensions we will also map to closely related types, -// e.g. "scss" will also return "sass". -// -func ResourceKeyPartitions(filename string) []string { - var partitions []string - filename = glob.NormalizePath(filename) - dir, name := path.Split(filename) - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(name)), ".") - - if dir != "" { - partitions = append(partitions, strings.Split(dir, "/")[0]) - } - - if ext != "" { - partitions = append(partitions, ext) - } - - if aliases, found := extAliasKeywords[ext]; found { - partitions = append(partitions, aliases...) - } - - if len(partitions) == 0 { - partitions = []string{CACHE_OTHER} - } - - return helpers.UniqueStringsSorted(partitions) } -// ResourceKeyContainsAny returns whether the key is a member of any of the -// given partitions. -// -// This is used for resource cache invalidation. -func ResourceKeyContainsAny(key string, partitions []string) bool { - parts := strings.Split(key, "/") - for _, p1 := range partitions { - for _, p2 := range parts { - if p1 == p2 { - return true - } - } - } - return false -} - -func newResourceCache(rs *Spec) *ResourceCache { +func newResourceCache(rs *Spec, memCache *memcache.Cache) *ResourceCache { return &ResourceCache{ rs: rs, fileCache: rs.FileCaches.AssetsCache(), - cache: make(map[string]interface{}), - nlocker: locker.NewLocker(), + cache: memCache.GetOrCreatePartition("resources", memcache.ClearOnChange), } } -func (c *ResourceCache) clear() { - c.Lock() - defer c.Unlock() - - c.cache = make(map[string]interface{}) - c.nlocker = locker.NewLocker() -} - -func (c *ResourceCache) Contains(key string) bool { - key = c.cleanKey(filepath.ToSlash(key)) - _, found := c.get(key) - return found -} - -func (c *ResourceCache) cleanKey(key string) string { - return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") -} - -func (c *ResourceCache) get(key string) (interface{}, bool) { - c.RLock() - defer c.RUnlock() - r, found := c.cache[key] - return r, found -} - -func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, error)) (resource.Resource, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) +func (c *ResourceCache) GetOrCreate(key string, clearWhen memcache.ClearWhen, f func() (resource.Resource, error)) (resource.Resource, error) { + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + ClearWhen: clearWhen, + } + }) if r == nil || err != nil { return nil, err } @@ -166,43 +61,19 @@ func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, err } func (c *ResourceCache) GetOrCreateResources(key string, f func() (resource.Resources, error)) (resource.Resources, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + } + }) if r == nil || err != nil { return nil, err } return r.(resource.Resources), nil } -func (c *ResourceCache) getOrCreate(key string, f func() (interface{}, error)) (interface{}, error) { - key = c.cleanKey(key) - // First check in-memory cache. - r, found := c.get(key) - if found { - return r, nil - } - // This is a potentially long running operation, so get a named lock. - c.nlocker.Lock(key) - - // Double check in-memory cache. - r, found = c.get(key) - if found { - c.nlocker.Unlock(key) - return r, nil - } - - defer c.nlocker.Unlock(key) - - r, err := f() - if err != nil { - return nil, err - } - - c.set(key, r) - - return r, nil - -} - func (c *ResourceCache) getFilenames(key string) (string, string) { filenameMeta := key + ".json" filenameContent := key + ".content" @@ -256,61 +127,7 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) } -func (c *ResourceCache) set(key string, r interface{}) { - c.Lock() - defer c.Unlock() - c.cache[key] = r -} - -func (c *ResourceCache) DeletePartitions(partitions ...string) { - partitionsSet := map[string]bool{ - // Always clear out the resources not matching any partition. - "other": true, - } - for _, p := range partitions { - partitionsSet[p] = true - } - - if partitionsSet[CACHE_CLEAR_ALL] { - c.clear() - return - } - - c.Lock() - defer c.Unlock() - - for k := range c.cache { - clear := false - for p := range partitionsSet { - if strings.Contains(k, p) { - // There will be some false positive, but that's fine. - clear = true - break - } - } - - if clear { - delete(c.cache, k) - } - } - -} - func (c *ResourceCache) DeleteContains(parts ...string) { - c.Lock() - defer c.Unlock() - - for k := range c.cache { - clear := false - for _, part := range parts { - if strings.Contains(k, part) { - clear = true - break - } - } - if clear { - delete(c.cache, k) - } - } + // TODO1 } diff --git a/resources/resource_cache_test.go b/resources/resource_cache_test.go deleted file mode 100644 index bcb24102594..00000000000 --- a/resources/resource_cache_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resources - -import ( - "path/filepath" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestResourceKeyPartitions(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - input string - expected []string - }{ - {"a.js", []string{"js"}}, - {"a.scss", []string{"sass", "scss"}}, - {"a.sass", []string{"sass", "scss"}}, - {"d/a.js", []string{"d", "js"}}, - {"js/a.js", []string{"js"}}, - {"D/a.JS", []string{"d", "js"}}, - {"d/a", []string{"d"}}, - {filepath.FromSlash("/d/a.js"), []string{"d", "js"}}, - {filepath.FromSlash("/d/e/a.js"), []string{"d", "js"}}, - } { - c.Assert(ResourceKeyPartitions(test.input), qt.DeepEquals, test.expected, qt.Commentf(test.input)) - } -} - -func TestResourceKeyContainsAny(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - key string - filename string - expected bool - }{ - {"styles/css", "asdf.css", true}, - {"styles/css", "styles/asdf.scss", true}, - {"js/foo.bar", "asdf.css", false}, - } { - c.Assert(ResourceKeyContainsAny(test.key, ResourceKeyPartitions(test.filename)), qt.Equals, test.expected) - } -} diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index 1ea92bea397..818dbaf5294 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -17,9 +17,10 @@ package bundler import ( "fmt" "io" - "path" "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" @@ -81,8 +82,7 @@ func (r *multiReadSeekCloser) Close() error { // Concat concatenates the list of Resource objects. func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resource, error) { - // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(targetPath, memcache.ClearOnRebuild, func() (resource.Resource, error) { var resolvedm media.Type // The given set of resources must be of the same Media Type. diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 4ac20d36e5a..49604ca0720 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -18,7 +18,8 @@ package create import ( "path" "path/filepath" - "strings" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/hugofs/glob" @@ -43,7 +44,7 @@ func New(rs *resources.Spec) *Client { // Get creates a new Resource by opening the given filename in the assets filesystem. func (c *Client) Get(filename string) (resource.Resource, error) { filename = filepath.Clean(filename) - return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(filename), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(filename), memcache.ClearOnChange, func() (resource.Resource, error) { return c.rs.New(resources.ResourceSourceDescriptor{ Fs: c.rs.BaseFs.Assets.Fs, LazyPublish: true, @@ -74,13 +75,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro name = "__match" } - pattern = glob.NormalizePath(pattern) - partitions := glob.FilterGlobParts(strings.Split(pattern, "/")) - if len(partitions) == 0 { - partitions = []string{resources.CACHE_OTHER} - } - key := path.Join(name, path.Join(partitions...)) - key = path.Join(key, pattern) + key := path.Join(name, glob.NormalizePath(pattern)) return c.rs.ResourceCache.GetOrCreateResources(key, func() (resource.Resources, error) { var res resource.Resources @@ -116,7 +111,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + r, err := c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(targetPath), memcache.ClearOnRebuild, func() (resource.Resource, error) { return c.rs.New( resources.ResourceSourceDescriptor{ Fs: c.rs.FileCaches.AssetsCache().Fs, @@ -128,4 +123,9 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro }) + if err == nil { + r.(resource.Staler).MarkStale() + } + return r, err + } diff --git a/resources/resource_metadata_test.go b/resources/resource_metadata_test.go index c79a5002121..0414127b92a 100644 --- a/resources/resource_metadata_test.go +++ b/resources/resource_metadata_test.go @@ -209,12 +209,12 @@ func TestAssignMetadata(t *testing.T) { }}, } { - foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) - logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) - foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) - logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) - foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) - logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) + foo2 = newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) + logo2 = newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) + foo1 = newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) + logo1 = newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) + foo3 = newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) + logo3 = newGenericResource(spec, nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) resources = resource.Resources{ foo2, diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 17225e3f5f9..2a4cfa26315 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/config" @@ -46,6 +48,7 @@ import ( func NewSpec( s *helpers.PathSpec, fileCaches filecache.Caches, + memCache *memcache.Cache, incr identity.Incrementer, logger loggers.Logger, errorHandler herrors.ErrorSender, @@ -89,11 +92,11 @@ func NewSpec( PostProcessResources: make(map[string]postpub.PostPublishedResource), imageCache: newImageCache( fileCaches.ImageCache(), - + memCache, s, )} - rs.ResourceCache = newResourceCache(rs) + rs.ResourceCache = newResourceCache(rs, memCache) return rs, nil @@ -129,57 +132,11 @@ func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { return r.newResourceFor(fd) } -func (r *Spec) CacheStats() string { - r.imageCache.mu.RLock() - defer r.imageCache.mu.RUnlock() - - s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) - - count := 0 - for k := range r.imageCache.store { - if count > 5 { - break - } - s += "\n" + k - count++ - } - - return s -} - -func (r *Spec) ClearCaches() { - r.imageCache.clear() - r.ResourceCache.clear() -} - -func (r *Spec) DeleteBySubstring(s string) { - r.imageCache.deleteIfContains(s) -} - func (s *Spec) String() string { return "spec" } // TODO(bep) clean up below -func (r *Spec) newGenericResource(sourceFs afero.Fs, - targetPathBuilder func() page.TargetPaths, - osFileInfo os.FileInfo, - sourceFilename, - baseFilename string, - mediaType media.Type) *genericResource { - return r.newGenericResourceWithBase( - sourceFs, - nil, - nil, - targetPathBuilder, - osFileInfo, - sourceFilename, - baseFilename, - mediaType, - ) - -} - func (r *Spec) newGenericResourceWithBase( sourceFs afero.Fs, openReadSeekerCloser resource.OpenReadSeekCloser, diff --git a/resources/resource_test.go b/resources/resource_test.go index 7a0b8069d79..75ac94f79ce 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -34,7 +34,7 @@ func TestGenericResource(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) - r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo.css") @@ -48,11 +48,11 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := newTargetPaths("/foo") - r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo/foo.css") - c.Assert(r.Key(), qt.Equals, "/foo/foo.css") + c.Assert(r.Key(), qt.Equals, "foo/foo.css") c.Assert(r.ResourceType(), qt.Equals, "css") } @@ -105,10 +105,10 @@ func TestResourcesByType(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), - spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo.png", "logo.css", pngType), + newGenericResource(spec, nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} c.Assert(len(resources.ByType("css")), qt.Equals, 3) c.Assert(len(resources.ByType("image")), qt.Equals, 1) @@ -119,11 +119,11 @@ func TestResourcesGetByPrefix(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} c.Assert(resources.GetMatch("asdf*"), qt.IsNil) c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -148,14 +148,14 @@ func TestResourcesGetMatch(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), } c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -212,7 +212,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := resource.Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)} + resources := resource.Resources{newGenericResource(spec, nil, nil, nil, "/a/"+a100, a100, media.CSSType)} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -228,17 +228,17 @@ func benchResources(b *testing.B) resource.Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) } return resources @@ -266,7 +266,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } b.StartTimer() diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 8eacf7da4e6..3d7af38c23d 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -16,6 +16,8 @@ package htesting import ( "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -51,7 +53,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 0462f7ecdfc..3f5d6905f06 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -4,6 +4,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/cache/memcache" + "image" "io" "io/ioutil" @@ -90,7 +92,9 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + mc := memcache.New(memcache.Config{}) + + spec, err := NewSpec(s, filecaches, mc, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -129,7 +133,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir @@ -207,3 +211,22 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { t.Fatalf("Failed to write file: %s", err) } } + +func newGenericResource(r *Spec, sourceFs afero.Fs, + targetPathBuilder func() page.TargetPaths, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + nil, + nil, + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} diff --git a/resources/transform.go b/resources/transform.go index 354a20eece7..74e29ed293e 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -18,9 +18,12 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/resources/images/exif" @@ -42,6 +45,8 @@ var ( _ resource.ContentResource = (*resourceAdapter)(nil) _ resource.ReadSeekCloserResource = (*resourceAdapter)(nil) _ resource.Resource = (*resourceAdapter)(nil) + _ resource.Staler = (*resourceAdapter)(nil) + _ resource.Staler = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) _ resource.ResourceMetaProvider = (*resourceAdapter)(nil) @@ -60,12 +65,17 @@ func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResour if lazyPublish { po = &publishOnce{} } + + s := &resourceOrigin{} + return &resourceAdapter{ resourceTransformations: &resourceTransformations{}, + resourceOrigin: s, resourceAdapterInner: &resourceAdapterInner{ - spec: spec, - publishOnce: po, - target: target, + spec: spec, + resourceOrigin: s, + publishOnce: po, + target: target, }, } } @@ -150,6 +160,11 @@ type publishOnce struct { type resourceAdapter struct { commonResource + + // This state is carried over into any clone of this adapter + // (when passed through a Hugo pipe). + *resourceOrigin + *resourceTransformations *resourceAdapterInner } @@ -188,8 +203,7 @@ func (r *resourceAdapter) Exif() *exif.Exif { } func (r *resourceAdapter) Key() string { - r.init(false, false) - return r.target.(resource.Identifier).Key() + return r.TransformationKey() } func (r *resourceAdapter) MediaType() media.Type { @@ -252,9 +266,10 @@ func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransfo } r.resourceAdapterInner = &resourceAdapterInner{ - spec: r.spec, - publishOnce: &publishOnce{}, - target: r.target, + spec: r.spec, + resourceOrigin: r.resourceOrigin, + publishOnce: &publishOnce{}, + target: r.target, } return &r, nil @@ -297,6 +312,57 @@ func (r *resourceAdapter) publish() { } func (r *resourceAdapter) TransformationKey() string { + r.transformationsKeyInit.Do(func() { + if len(r.transformations) == 0 { + r.transformationsKey = r.target.Key() + return + } + + var adder string + for _, tr := range r.transformations { + adder = adder + "_" + tr.Key().Value() + } + + key := r.target.Key() + adder = "_" + helpers.MD5String(adder) + + // Preserve any file extension if possible. + dotIdx := strings.LastIndex(key, ".") + if dotIdx == -1 { + key += adder + } else { + key = key[:dotIdx] + adder + key[dotIdx:] + } + + key = memcache.CleanKey(key) + r.transformationsKey = key + }) + + return r.transformationsKey +} + +// We changed the format of the resource cache keys in Hugo v0.75. +// To reduce the nois, especially on the theme site, we fall back to reading +// files on the old format. +// TODO(bep) eventually remove. +func (r *resourceAdapter) transformationKeyV074() string { + cleanKey := func(key string) string { + return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") + } + + resourceKeyPartition := func(filename string) string { + ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") + if ext == "" { + ext = "other" + } + return ext + } + + resourceCacheKey := func(filename string) string { + filename = filepath.ToSlash(filename) + return path.Join(resourceKeyPartition(filename), filename) + } + // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. var key string @@ -304,35 +370,33 @@ func (r *resourceAdapter) TransformationKey() string { key = key + "_" + tr.Key().Value() } - base := ResourceCacheKey(r.target.Key()) - return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key) -} + base := resourceCacheKey(r.target.RelPermalink()) + return cleanKey(base) + "_" + helpers.MD5String(key) -func (r *resourceAdapter) transform(publish, setContent bool) error { - cache := r.spec.ResourceCache +} +func (r *resourceAdapter) getOrTransform(publish, setContent bool) error { key := r.TransformationKey() + res, err := r.spec.ResourceCache.cache.GetOrCreate(key, func() memcache.Entry { + r, err := r.transform(key, publish, setContent) + return memcache.Entry{ + Value: r, + Err: err, + } + }) - cached, found := cache.get(key) - - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - return nil + if err != nil { + return err } - // Acquire a write lock for the named transformation. - cache.nlocker.Lock(key) - // Check the cache again. - cached, found = cache.get(key) - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - cache.nlocker.Unlock(key) - return nil - } + r.resourceAdapterInner = res.(*resourceAdapterInner) - defer cache.nlocker.Unlock(key) - defer cache.set(key, r.resourceAdapterInner) + return nil +} + +func (r *resourceAdapter) transform(key string, publish, setContent bool) (*resourceAdapterInner, error) { + cache := r.spec.ResourceCache b1 := bp.GetBuffer() b2 := bp.GetBuffer() defer bp.PutBuffer(b1) @@ -353,7 +417,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { contentrc, err := contentReadSeekerCloser(r.target) if err != nil { - return err + return nil, err } defer contentrc.Close() @@ -426,21 +490,25 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { } else { err = tr.Transform(tctx) if err != nil && err != herrors.ErrFeatureNotAvailable { - return newErr(err) + return nil, newErr(err) } if mayBeCachedOnDisk { tryFileCache = r.spec.BuildConfig.UseResourceCache(err) } if err != nil && !tryFileCache { - return newErr(err) + return nil, newErr(err) } } if tryFileCache { f := r.target.tryTransformedFileCache(key, updates) if f == nil { - return newErr(errors.Errorf("resource %q not found in file cache", key)) + keyOldFormat := r.transformationKeyV074() + f = r.target.tryTransformedFileCache(keyOldFormat, updates) + if f == nil { + return nil, newErr(errors.Errorf("resource %q not found in file cache", key)) + } } transformedContentr = f updates.sourceFs = cache.fileCache.Fs @@ -465,7 +533,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { if publish { publicw, err := r.target.openPublishFileForWriting(updates.targetPath) if err != nil { - return err + return nil, err } publishwriters = append(publishwriters, publicw) } @@ -475,7 +543,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { // Also write it to the cache fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata()) if err != nil { - return err + return nil, err } updates.sourceFilename = &fi.Name updates.sourceFs = cache.fileCache.Fs @@ -506,7 +574,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { publishw := hugio.NewMultiWriteCloser(publishwriters...) _, err = io.Copy(publishw, transformedContentr) if err != nil { - return err + return nil, err } publishw.Close() @@ -517,11 +585,11 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { newTarget, err := r.target.cloneWithUpdates(updates) if err != nil { - return err + return nil, err } r.target = newTarget - return nil + return r.resourceAdapterInner, nil } func (r *resourceAdapter) init(publish, setContent bool) { @@ -541,7 +609,7 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { r.publishOnce = nil } - r.transformationsErr = r.transform(publish, setContent) + r.transformationsErr = r.getOrTransform(publish, setContent) if r.transformationsErr != nil { if r.spec.ErrorSender != nil { r.spec.ErrorSender.SendError(r.transformationsErr) @@ -559,16 +627,24 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { type resourceAdapterInner struct { target transformableResource + *resourceOrigin + spec *Spec // Handles publishing (to /public) if needed. *publishOnce } +func (r *resourceAdapterInner) ResourceTarget() resource.Resource { + return r.target +} + type resourceTransformations struct { - transformationsInit sync.Once - transformationsErr error - transformations []ResourceTransformation + transformationsInit sync.Once + transformationsErr error + transformationsKeyInit sync.Once + transformationsKey string + transformations []ResourceTransformation } type transformableResource interface { @@ -603,7 +679,6 @@ func (u *transformationUpdate) toTransformedResourceMetadata() transformedResour } func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) { - u.targetPath = ctx.OutPath u.mediaType = ctx.OutMediaType u.data = ctx.Data u.targetPath = ctx.InPath diff --git a/tpl/openapi/openapi3/openapi3.go b/tpl/openapi/openapi3/openapi3.go index 7dfd2f6a797..a95e288e1a4 100644 --- a/tpl/openapi/openapi3/openapi3.go +++ b/tpl/openapi/openapi3/openapi3.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" kopenapi3 "github.com/getkin/kin-openapi/openapi3" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/resources/resource" @@ -29,22 +29,15 @@ import ( // New returns a new instance of the openapi3-namespaced template functions. func New(deps *deps.Deps) *Namespace { - // TODO1 consolidate when merging that "other branch" -- but be aware of the keys. - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - return &Namespace{ - cache: cache, + cache: deps.MemCache.GetOrCreatePartition("tpl/openapi3", memcache.ClearOnChange), deps: deps, } } // Namespace provides template functions for the "openapi3". type Namespace struct { - cache *namedmemcache.Cache + cache memcache.Getter deps *deps.Deps } @@ -55,21 +48,22 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg return nil, errors.New("no Key set in Resource") } - v, err := ns.cache.GetOrCreate(key, func() (interface{}, error) { + v, err := ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } + defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } s := &kopenapi3.Swagger{} @@ -80,12 +74,14 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg err = metadecoders.Default.UnmarshalTo(b, f, s) } if err != nil { - return nil, err + return memcache.Entry{Err: err} } err = kopenapi3.NewSwaggerLoader().ResolveRefsIn(s, nil) - return s, err + return memcache.Entry{Value: s, Err: err, StaleFunc: func() bool { + return resource.IsStaleAny(r) + }} }) if err != nil { diff --git a/tpl/transform/init_test.go b/tpl/transform/init_test.go index 47bd8a391e6..393b6b140d8 100644 --- a/tpl/transform/init_test.go +++ b/tpl/transform/init_test.go @@ -16,6 +16,8 @@ package transform import ( "testing" + "github.com/gohugoio/hugo/cache/memcache" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/htesting/hqt" @@ -28,7 +30,9 @@ func TestInit(t *testing.T) { var ns *internal.TemplateFuncsNamespace for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns = nsf(&deps.Deps{}) + ns = nsf(&deps.Deps{ + MemCache: memcache.New(memcache.Config{}), + }) if ns.Name == name { found = true break diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index b168d2a50d4..e4ea17beba2 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -18,7 +18,7 @@ import ( "html" "html/template" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" @@ -27,22 +27,19 @@ import ( // New returns a new instance of the transform-namespaced template functions. func New(deps *deps.Deps) *Namespace { - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - + if deps.MemCache == nil { + panic("must provide MemCache") + } return &Namespace{ - cache: cache, deps: deps, + cache: deps.MemCache.GetOrCreatePartition("tpl/transform", memcache.ClearOnChange), } } // Namespace provides template functions for the "transform" namespace. type Namespace struct { - cache *namedmemcache.Cache deps *deps.Deps + cache memcache.Getter } // Emojify returns a copy of s with all emoji codes replaced with actual emojis. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index b3f4206ff6b..1a14661f246 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -16,6 +16,8 @@ package transform import ( "html/template" + "github.com/gohugoio/hugo/cache/memcache" + "testing" "github.com/gohugoio/hugo/common/loggers" @@ -251,6 +253,7 @@ func newDeps(cfg config.Provider) *deps.Deps { return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), + MemCache: memcache.New(memcache.Config{}), ContentSpec: cs, } } diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index b606c870aa7..ea43f6d0ff9 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -17,6 +17,8 @@ import ( "io/ioutil" "strings" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/common/types" @@ -69,24 +71,28 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key += decoder.OptionsKey() } - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } - return decoder.Unmarshal(b, f) + v, err := decoder.Unmarshal(b, f) + + return memcache.Entry{Value: v, Err: err, StaleFunc: func() bool { + return resource.IsStaleAny(r) + }} }) } @@ -97,13 +103,15 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key := helpers.MD5String(dataStr) - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := decoder.FormatFromContentString(dataStr) if f == "" { - return nil, errors.New("unknown format") + return memcache.Entry{Err: errors.New("unknown format")} } - return decoder.Unmarshal([]byte(dataStr), f) + v, err := decoder.Unmarshal([]byte(dataStr), f) + + return memcache.Entry{Value: v, Err: err} }) } diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 183bdefd5bc..8796cef3724 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -145,7 +145,7 @@ a;b;c`, mime: media.CSVType}, map[string]interface{}{"DElimiter": ";", "Comment" {tstNoStringer{}, nil, false}, } { - ns.cache.Clear() + ns.deps.MemCache.Clear() var args []interface{}