From 033f051ce6b875257a9ddf1af95ff1751ff894ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 25 Jun 2020 11:32:42 +0200 Subject: [PATCH] Add central LRU cache that adjusts to available memory Hugo has always been a active user of in-memory caches, but before this commit we did nothing to control the memory usage. One failing example would be loading lots of big JSON data files and unmarshal them via `transform.Unmarshal`. This commit consolidates all these caches into one single LRU cache with an eviction strategy that also considers used vs. available memory. Hugo will try to limit its memory usage to 1/4 or total system memory, but this can be controlled with the `HUGO_MEMORYLIMIT` environment variable (a float value representing Gigabytes). A natural next step after this would be to use this cache for `.Content`. Fixes #7425 Fixes #7437 Fixes #7436 Fixes #7882 Updates #7544 --- cache/memcache/memcache.go | 506 ++++++++++++++++++ cache/memcache/memcache_test.go | 178 ++++++ cache/namedmemcache/named_cache.go | 79 --- cache/namedmemcache/named_cache_test.go | 80 --- commands/commands.go | 2 +- commands/hugo.go | 22 +- compare/compare.go | 2 +- config/env.go | 35 ++ config/env_test.go | 2 + deps/deps.go | 20 +- .../en/getting-started/configuration.md | 9 + go.mod | 2 + go.sum | 116 +--- helpers/general.go | 18 + hugofs/fileinfo.go | 5 + hugofs/rootmapping_fs.go | 1 + hugolib/content_map_page.go | 7 + hugolib/filesystems/basefs.go | 73 ++- hugolib/hugo_sites.go | 25 +- hugolib/hugo_sites_build.go | 4 + hugolib/hugo_sites_build_test.go | 123 ++++- hugolib/hugo_sites_multihost_test.go | 2 +- hugolib/page.go | 14 +- hugolib/pagebundler_test.go | 4 +- hugolib/pages_capture.go | 4 - hugolib/paths/paths.go | 6 +- hugolib/resource_chain_test.go | 2 +- hugolib/resource_change_test.go | 322 +++++++++++ hugolib/site.go | 155 +++--- hugolib/testhelpers_test.go | 27 +- identity/identity.go | 79 ++- identity/identity_test.go | 19 + identity/pathIdentity.go | 167 ++++++ identity/pathIdentity_test.go | 36 ++ markup/converter/converter.go | 2 +- modules/client.go | 2 +- resources/image.go | 4 +- resources/image_cache.go | 166 +++--- resources/image_test.go | 2 +- resources/page/page_nop.go | 10 +- resources/page/pages.go | 11 + resources/page/testhelpers_test.go | 8 + resources/resource.go | 63 ++- resources/resource/resources.go | 14 + resources/resource/resourcetypes.go | 35 ++ resources/resource_cache.go | 228 +------- resources/resource_cache_test.go | 58 -- .../resource_factories/bundler/bundler.go | 6 +- resources/resource_factories/create/create.go | 20 +- resources/resource_metadata_test.go | 12 +- resources/resource_spec.go | 52 +- resources/resource_test.go | 50 +- .../htesting/testhelpers.go | 4 +- resources/testhelpers_test.go | 27 +- resources/transform.go | 171 ++++-- tpl/openapi/openapi3/openapi3.go | 33 +- tpl/tplimpl/template.go | 4 +- tpl/transform/init_test.go | 6 +- tpl/transform/transform.go | 15 +- tpl/transform/transform_test.go | 3 + tpl/transform/unmarshal.go | 29 +- tpl/transform/unmarshal_test.go | 2 +- 62 files changed, 2194 insertions(+), 989 deletions(-) create mode 100644 cache/memcache/memcache.go create mode 100644 cache/memcache/memcache_test.go delete mode 100644 cache/namedmemcache/named_cache.go delete mode 100644 cache/namedmemcache/named_cache_test.go create mode 100644 hugolib/resource_change_test.go create mode 100644 identity/pathIdentity.go create mode 100644 identity/pathIdentity_test.go delete mode 100644 resources/resource_cache_test.go diff --git a/cache/memcache/memcache.go b/cache/memcache/memcache.go new file mode 100644 index 00000000000..be5d65ab943 --- /dev/null +++ b/cache/memcache/memcache.go @@ -0,0 +1,506 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memcache provides the core memory cache used in Hugo. +package memcache + +import ( + "math" + "path" + "regexp" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gohugoio/hugo/identity" + + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/resources/resource" + + "github.com/gohugoio/hugo/helpers" + + "github.com/BurntSushi/locker" + "github.com/karlseguin/ccache/v2" +) + +const ( + ClearOnRebuild ClearWhen = iota + 1 + ClearOnChange + ClearNever +) + +const ( + cacheVirtualRoot = "_root/" +) + +var ( + + // Consider a change in files matching this expression a "JS change". + isJSFileRe = regexp.MustCompile(`\.(js|ts|jsx|tsx)`) + + // Consider a change in files matching this expression a "CSS change". + isCSSFileRe = regexp.MustCompile(`\.(css|scss|sass)`) + + // These config files are tightly related to CSS editing, so consider + // a change to any of them a "CSS change". + isCSSConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`) +) + +const unknownExtension = "unkn" + +// New creates a new cache. +func New(conf Config) *Cache { + if conf.TTL == 0 { + conf.TTL = time.Second * 33 + } + if conf.CheckInterval == 0 { + conf.CheckInterval = time.Second * 2 + } + if conf.MaxSize == 0 { + conf.MaxSize = 100000 + } + if conf.ItemsToPrune == 0 { + conf.ItemsToPrune = 200 + } + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + stats := &stats{ + memstatsStart: m, + maxSize: conf.MaxSize, + availableMemory: config.GetMemoryLimit(), + } + + conf.MaxSize = stats.adjustMaxSize(nil) + + c := &Cache{ + conf: conf, + cache: ccache.Layered(ccache.Configure().MaxSize(conf.MaxSize).ItemsToPrune(conf.ItemsToPrune)), + getters: make(map[string]*getter), + ttl: conf.TTL, + stats: stats, + nlocker: locker.NewLocker(), + } + + c.stop = c.start() + + return c +} + +// CleanKey turns s into a format suitable for a cache key for this package. +// The key will be a Unix-styled path without any leading slash. +// If the input string does not contain any slash, a root will be prepended. +// If the input string does not contain any ".", a dummy file suffix will be appended. +// These are to make sure that they can effectively partake in the "cache cleaning" +// strategy used in server mode. +func CleanKey(s string) string { + s = path.Clean(helpers.ToSlashTrimLeading(s)) + if !strings.ContainsRune(s, '/') { + s = cacheVirtualRoot + s + } + if !strings.ContainsRune(s, '.') { + s += "." + unknownExtension + } + + return s +} + +// InsertKeyPathElement inserts the given element after the first '/' in key. +func InsertKeyPathElements(key string, elements ...string) string { + slashIdx := strings.Index(key, "/") + return key[:slashIdx] + "/" + path.Join(elements...) + key[slashIdx:] +} + +// Cache configures a cache. +type Cache struct { + mu sync.Mutex + getters map[string]*getter + + conf Config + cache *ccache.LayeredCache + + ttl time.Duration + nlocker *locker.Locker + + stats *stats + stopOnce sync.Once + stop func() +} + +// Clear clears the cache state. +// This method is not thread safe. +func (c *Cache) Clear() { + c.nlocker = locker.NewLocker() + for _, g := range c.getters { + g.c.DeleteAll(g.partition) + } +} + +// ClearOn clears all the caches given a eviction strategy and (optional) a +// change set. +// This method is not thread safe. +func (c *Cache) ClearOn(when ClearWhen, changeset ...identity.PathIdentity) { + if when == 0 { + panic("invalid ClearWhen") + } + + for _, g := range c.getters { + if g.clearWhen == ClearNever { + continue + } + + if g.clearWhen == when { + // Clear all. + g.Clear() + continue + } + + shouldDelete := func(key string, e Entry) bool { + + // We always clear elements marked as stale. + if resource.IsStaleAny(e, e.Value) { + return true + } + + if e.ClearWhen == ClearNever { + return false + } + + if e.ClearWhen == when && e.ClearWhen == ClearOnRebuild { + return true + } + + // Now check if this entry has changed based on the changeset + // based on filesystem events. + + if len(changeset) == 0 { + // Nothing changed. + return false + } + + for _, id := range changeset { + path := CleanKey(id.Path()) + + prefix, _ := splitBasePathAndExt(path) + if prefix == "" { + prefix = cacheVirtualRoot + } + + // Will clear out all files that share a common root path, + // e.g "styles" in "styles/main.css". + if strings.HasPrefix(key, prefix) { + //return true + } + + switch v := e.Value.(type) { + case identity.IsNotDependentProvider: + if !v.IsNotDependent(id) { + return true + } + case resource.MediaTypeProvider: + // TODO1 move this, re up. + // Clear out all files sharing the same or related MIME type. + isJS := isJSFileRe.MatchString(path) + var isCSS bool + if !isJS { + isCSS = isCSSFileRe.MatchString(path) || isCSSConfigRe.MatchString(path) + } + + if isJS && isJSType(v.MediaType()) { + return true + } + + if isCSS && isCSSType(v.MediaType()) { + return true + } + + } + } + + // Keep it. + return false + } + + // Two passes, the last one to catch any leftover values marked stale in the first. + g.c.cache.DeleteFunc(g.partition, func(key string, item *ccache.Item) bool { + e := item.Value().(Entry) + if shouldDelete(key, e) { + resource.MarkStale(e.Value) + return true + } + return false + }) + + g.c.cache.DeleteFunc(g.partition, func(key string, item *ccache.Item) bool { + e := item.Value().(Entry) + return resource.IsStaleAny(e.Value) + }) + + } +} + +/* + assets: css/mystyles.scss + content: blog/mybundle/data.json +*/ +func shouldEvict(key string, e Entry, when ClearWhen, changeset ...identity.PathIdentity) bool { + return false +} + +func (c *Cache) DeleteAll(primary string) bool { + return c.cache.DeleteAll(primary) +} + +func (c *Cache) GetDropped() int { + return c.cache.GetDropped() +} + +func (c *Cache) GetOrCreatePartition(partition string, clearWhen ClearWhen) Getter { + if clearWhen == 0 { + panic("GetOrCreatePartition: invalid ClearWhen") + } + c.mu.Lock() + defer c.mu.Unlock() + + g, found := c.getters[partition] + if found { + if g.clearWhen != clearWhen { + panic("GetOrCreatePartition called with the same partition but different clearing strategy.") + } + return g + } + + g = &getter{ + partition: partition, + c: c, + clearWhen: clearWhen, + } + + c.getters[partition] = g + + return g +} + +func (c *Cache) Stop() { + c.stopOnce.Do(func() { + c.stop() + c.cache.Stop() + }) +} + +func (c *Cache) start() func() { + ticker := time.NewTicker(c.conf.CheckInterval) + quit := make(chan struct{}) + + checkAndAdjustMaxSize := func() { + var m runtime.MemStats + cacheDropped := c.GetDropped() + c.stats.decr(cacheDropped) + + runtime.ReadMemStats(&m) + c.stats.memstatsCurrent = m + c.stats.adjustMaxSize(c.cache.SetMaxSize) + + //fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped) + + } + go func() { + for { + select { + case <-ticker.C: + checkAndAdjustMaxSize() + case <-quit: + ticker.Stop() + return + } + } + }() + + return func() { + close(quit) + } +} + +// getOrCreate tries to get the value with the given cache paths, if not found +// create will be called and the result cached. +// +// This method is thread safe. +func (c *Cache) getOrCreate(primary, secondary string, create func() Entry) (interface{}, error) { + + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // The provided create function may be a relatively time consuming operation, + // and there will in the commmon case be concurrent requests for the same key'd + // resource, so make sure we pause these until the result is ready. + path := primary + secondary + c.nlocker.Lock(path) + defer c.nlocker.Unlock(path) + + // Try again. + if v := c.cache.Get(primary, secondary); v != nil { + e := v.Value().(Entry) + if !resource.IsStaleAny(e, e.Value) { + return e.Value, e.Err + } + } + + // Create it and store it in cache. + entry := create() + + if entry.Err != nil { + entry.ClearWhen = ClearOnRebuild + } else if entry.ClearWhen == 0 { + panic("entry: invalid ClearWhen") + } + + entry.size = 1 // For now. + + c.cache.Set(primary, secondary, entry, c.ttl) + c.stats.incr(1) + + return entry.Value, entry.Err +} + +type ClearWhen int + +type Config struct { + CheckInterval time.Duration + MaxSize int64 + ItemsToPrune uint32 + TTL time.Duration + Running bool +} + +type Entry struct { + Value interface{} + size int64 + Err error + StaleFunc func() bool + ClearWhen +} + +func (e Entry) Size() int64 { + return e.size +} + +func (e Entry) IsStale() bool { + return e.StaleFunc != nil && e.StaleFunc() +} + +type Getter interface { + Clear() + GetOrCreate(path string, create func() Entry) (interface{}, error) +} + +type getter struct { + c *Cache + partition string + + clearWhen ClearWhen +} + +func (g *getter) Clear() { + g.c.DeleteAll(g.partition) +} + +func (g *getter) GetOrCreate(path string, create func() Entry) (interface{}, error) { + return g.c.getOrCreate(g.partition, path, create) +} + +type stats struct { + memstatsStart runtime.MemStats + memstatsCurrent runtime.MemStats + maxSize int64 + availableMemory uint64 + numItems uint64 +} + +func (s *stats) getNumItems() uint64 { + return atomic.LoadUint64(&s.numItems) +} + +func (s *stats) adjustMaxSize(setter func(size int64)) int64 { + newSize := int64(float64(s.maxSize) * s.resizeFactor()) + if newSize != s.maxSize && setter != nil { + setter(newSize) + } + return newSize +} + +func (s *stats) decr(i int) { + atomic.AddUint64(&s.numItems, ^uint64(i-1)) +} + +func (s *stats) incr(i int) { + atomic.AddUint64(&s.numItems, uint64(i)) +} + +func (s *stats) resizeFactor() float64 { + if s.memstatsCurrent.Alloc == 0 { + return 1.0 + } + return math.Floor(float64(s.availableMemory/s.memstatsCurrent.Alloc)*10) / 10 +} + +// Helpers to help eviction of related media types. +func isCSSType(m media.Type) bool { + tp := m.Type() + return tp == media.CSSType.Type() || tp == media.SASSType.Type() || tp == media.SCSSType.Type() +} + +func isJSType(m media.Type) bool { + tp := m.Type() + return tp == media.JavascriptType.Type() || tp == media.TypeScriptType.Type() || tp == media.JSXType.Type() || tp == media.TSXType.Type() +} + +func keyValid(s string) bool { + if len(s) < 5 { + return false + } + if strings.ContainsRune(s, '\\') { + return false + } + if strings.HasPrefix(s, "/") { + return false + } + if !strings.ContainsRune(s, '/') { + return false + } + + dotIdx := strings.Index(s, ".") + if dotIdx == -1 || dotIdx == len(s)-1 { + return false + } + + return true +} + +// This assumes a valid key path. +func splitBasePathAndExt(path string) (string, string) { + dotIdx := strings.LastIndex(path, ".") + ext := path[dotIdx+1:] + slashIdx := strings.Index(path, "/") + + return path[:slashIdx], ext +} diff --git a/cache/memcache/memcache_test.go b/cache/memcache/memcache_test.go new file mode 100644 index 00000000000..99bb9c30a44 --- /dev/null +++ b/cache/memcache/memcache_test.go @@ -0,0 +1,178 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memcache + +import ( + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + qt "github.com/frankban/quicktest" +) + +func TestCache(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{}) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + a := cache.GetOrCreatePartition("a", ClearNever) + + for i := 0; i < 5; i++ { + v1, err := a.GetOrCreate("a1", create) + c.Assert(err, qt.IsNil) + c.Assert(v1, qt.Equals, 1) + v2, err := a.GetOrCreate("a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v2, qt.Equals, 2) + } + + cache.Clear() + + v3, err := a.GetOrCreate("a2", create) + c.Assert(err, qt.IsNil) + c.Assert(v3, qt.Equals, 3) +} + +func TestCacheConcurrent(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + var wg sync.WaitGroup + + cache := New(Config{}) + + create := func(i int) func() Entry { + return func() Entry { + return Entry{Value: i} + } + } + + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + id := fmt.Sprintf("id%d", j) + v, err := cache.getOrCreate("a", id, create(j)) + c.Assert(err, qt.IsNil) + c.Assert(v, qt.Equals, j) + } + }() + } + wg.Wait() +} + +func TestCacheMemStats(t *testing.T) { + t.Parallel() + c := qt.New(t) + + cache := New(Config{ + ItemsToPrune: 10, + CheckInterval: 500 * time.Millisecond, + }) + + s := cache.stats + + c.Assert(s.memstatsStart.Alloc > 0, qt.Equals, true) + c.Assert(s.memstatsCurrent.Alloc, qt.Equals, uint64(0)) + c.Assert(s.availableMemory > 0, qt.Equals, true) + c.Assert(s.numItems, qt.Equals, uint64(0)) + + counter := 0 + create := func() Entry { + counter++ + return Entry{Value: counter} + } + + for i := 1; i <= 20; i++ { + _, err := cache.getOrCreate("a", fmt.Sprintf("b%d", i), create) + c.Assert(err, qt.IsNil) + } + + c.Assert(s.getNumItems(), qt.Equals, uint64(20)) + cache.cache.SetMaxSize(10) + time.Sleep(time.Millisecond * 600) + c.Assert(int(s.getNumItems()), qt.Equals, 10) + +} + +func TestSplitBasePathAndExt(t *testing.T) { + t.Parallel() + c := qt.New(t) + + tests := []struct { + path string + a string + b string + }{ + {"a/b.json", "a", "json"}, + {"a/b/c/d.json", "a", "json"}, + } + for i, this := range tests { + msg := qt.Commentf("test %d", i) + a, b := splitBasePathAndExt(this.path) + + c.Assert(a, qt.Equals, this.a, msg) + c.Assert(b, qt.Equals, this.b, msg) + } + +} + +func TestCleanKey(t *testing.T) { + c := qt.New(t) + + c.Assert(CleanKey(filepath.FromSlash("a/b/c.js")), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a//b////c.js"), qt.Equals, "a/b/c.js") + c.Assert(CleanKey("a.js"), qt.Equals, "_root/a.js") + c.Assert(CleanKey("b/a"), qt.Equals, "b/a.unkn") + +} + +func TestKeyValid(t *testing.T) { + c := qt.New(t) + + c.Assert(keyValid("a/b.j"), qt.Equals, true) + c.Assert(keyValid("a/b."), qt.Equals, false) + c.Assert(keyValid("a/b"), qt.Equals, false) + c.Assert(keyValid("/a/b.txt"), qt.Equals, false) + c.Assert(keyValid("a\\b.js"), qt.Equals, false) + +} + +func TestInsertKeyPathElement(t *testing.T) { + c := qt.New(t) + + c.Assert(InsertKeyPathElements("a/b.j", "en"), qt.Equals, "a/en/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "en", "foo"), qt.Equals, "a/en/foo/b.j") + c.Assert(InsertKeyPathElements("a/b.j", "", "foo"), qt.Equals, "a/foo/b.j") + +} + +func TestShouldEvict(t *testing.T) { + // TODO1 remove? + //c := qt.New(t) + + //fmt.Println("=>", CleanKey("kkk")) + //c.Assert(shouldEvict("key", Entry{}, ClearNever, identity.NewPathIdentity(files.ComponentFolderAssets, "a/b/c.js")), qt.Equals, true) +} diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go deleted file mode 100644 index d8c229a013b..00000000000 --- a/cache/namedmemcache/named_cache.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package namedmemcache provides a memory cache with a named lock. This is suitable -// for situations where creating the cached resource can be time consuming or otherwise -// resource hungry, or in situations where a "once only per key" is a requirement. -package namedmemcache - -import ( - "sync" - - "github.com/BurntSushi/locker" -) - -// Cache holds the cached values. -type Cache struct { - nlocker *locker.Locker - cache map[string]cacheEntry - mu sync.RWMutex -} - -type cacheEntry struct { - value interface{} - err error -} - -// New creates a new cache. -func New() *Cache { - return &Cache{ - nlocker: locker.NewLocker(), - cache: make(map[string]cacheEntry), - } -} - -// Clear clears the cache state. -func (c *Cache) Clear() { - c.mu.Lock() - defer c.mu.Unlock() - - c.cache = make(map[string]cacheEntry) - c.nlocker = locker.NewLocker() - -} - -// GetOrCreate tries to get the value with the given cache key, if not found -// create will be called and cached. -// This method is thread safe. It also guarantees that the create func for a given -// key is invoced only once for this cache. -func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) { - c.mu.RLock() - entry, found := c.cache[key] - c.mu.RUnlock() - - if found { - return entry.value, entry.err - } - - c.nlocker.Lock(key) - defer c.nlocker.Unlock(key) - - // Create it. - value, err := create() - - c.mu.Lock() - c.cache[key] = cacheEntry{value: value, err: err} - c.mu.Unlock() - - return value, err -} diff --git a/cache/namedmemcache/named_cache_test.go b/cache/namedmemcache/named_cache_test.go deleted file mode 100644 index 9feddb11f2a..00000000000 --- a/cache/namedmemcache/named_cache_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package namedmemcache - -import ( - "fmt" - "sync" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestNamedCache(t *testing.T) { - t.Parallel() - c := qt.New(t) - - cache := New() - - counter := 0 - create := func() (interface{}, error) { - counter++ - return counter, nil - } - - for i := 0; i < 5; i++ { - v1, err := cache.GetOrCreate("a1", create) - c.Assert(err, qt.IsNil) - c.Assert(v1, qt.Equals, 1) - v2, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v2, qt.Equals, 2) - } - - cache.Clear() - - v3, err := cache.GetOrCreate("a2", create) - c.Assert(err, qt.IsNil) - c.Assert(v3, qt.Equals, 3) -} - -func TestNamedCacheConcurrent(t *testing.T) { - t.Parallel() - - c := qt.New(t) - - var wg sync.WaitGroup - - cache := New() - - create := func(i int) func() (interface{}, error) { - return func() (interface{}, error) { - return i, nil - } - } - - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - id := fmt.Sprintf("id%d", j) - v, err := cache.GetOrCreate(id, create(j)) - c.Assert(err, qt.IsNil) - c.Assert(v, qt.Equals, j) - } - }() - } - wg.Wait() -} diff --git a/commands/commands.go b/commands/commands.go index ce5f0ff7d97..bbbbc5aba4d 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -301,7 +301,7 @@ func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) { cmd.Flags().BoolP("path-warnings", "", false, "print warnings on duplicate target paths etc.") cmd.Flags().StringVarP(&cc.cpuprofile, "profile-cpu", "", "", "write cpu profile to `file`") cmd.Flags().StringVarP(&cc.memprofile, "profile-mem", "", "", "write memory profile to `file`") - cmd.Flags().BoolVarP(&cc.printm, "print-mem", "", false, "print memory usage to screen at intervals") + cmd.Flags().BoolVarP(&cc.printm, "printMem", "", false, "print memory usage to screen at intervals") cmd.Flags().StringVarP(&cc.mutexprofile, "profile-mutex", "", "", "write Mutex profile to `file`") cmd.Flags().StringVarP(&cc.traceprofile, "trace", "", "", "write trace to `file` (not useful in general)") diff --git a/commands/hugo.go b/commands/hugo.go index 58f33b7752e..74fc184c82e 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -427,8 +427,14 @@ func (c *commandeer) initMemTicker() func() { quit := make(chan struct{}) printMem := func() { var m runtime.MemStats + var cacheDropped int + h := c.hugo() + if h != nil && h.MemCache != nil { + cacheDropped = h.MemCache.GetDropped() + } + runtime.ReadMemStats(&m) - fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\n\n", formatByteCount(m.Alloc), formatByteCount(m.TotalAlloc), formatByteCount(m.Sys), m.NumGC) + fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\nConfiguredMemoryLimit = %v\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped, helpers.FormatByteCount(config.GetMemoryLimit())) } @@ -1211,17 +1217,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { return name } - -func formatByteCount(b uint64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", - float64(b)/float64(div), "kMGTPE"[exp]) -} diff --git a/compare/compare.go b/compare/compare.go index 537a6667638..b7a98dfcbcf 100644 --- a/compare/compare.go +++ b/compare/compare.go @@ -14,7 +14,7 @@ package compare // Eqer can be used to determine if this value is equal to the other. -// The semantics of equals is that the two value are interchangeable +// The semantics of equals is that the two values are interchangeable // in the Hugo templates. type Eqer interface { Eq(other interface{}) bool diff --git a/config/env.go b/config/env.go index f482cd24733..06233c4b641 100644 --- a/config/env.go +++ b/config/env.go @@ -17,6 +17,9 @@ import ( "os" "runtime" "strconv" + + "github.com/pbnjay/memory" + "strings" ) @@ -33,6 +36,38 @@ func GetNumWorkerMultiplier() int { return runtime.NumCPU() } +const ( + gigabyte = 1 << 30 +) + +// GetMemoryLimit returns the upper memory limit in bytes for Hugo's in-memory caches. +// Note that this does not represent "all of the memory" that Hugo will use, +// so it needs to be set to a lower number than the available system memory. +// It will read from the HUGO_MEMORYLIMIT (in Gigabytes) environment variable. +// If that is not set, it will set aside a quarter of the total system memory. +func GetMemoryLimit() uint64 { + if mem := os.Getenv("HUGO_MEMORYLIMIT"); mem != "" { + if v := stringToGibabyte(mem); v > 0 { + return v + } + + } + + m := memory.TotalMemory() + if m != 0 { + return uint64(m / 4) + } + + return 2 * gigabyte +} + +func stringToGibabyte(f string) uint64 { + if v, err := strconv.ParseFloat(f, 32); err == nil && v > 0 { + return uint64(v * gigabyte) + } + return 0 +} + // SetEnvVars sets vars on the form key=value in the oldVars slice. func SetEnvVars(oldVars *[]string, keyValues ...string) { for i := 0; i < len(keyValues); i += 2 { diff --git a/config/env_test.go b/config/env_test.go index 3c402b9ef8d..3b0faac3b7d 100644 --- a/config/env_test.go +++ b/config/env_test.go @@ -29,4 +29,6 @@ func TestSetEnvVars(t *testing.T) { key, val := SplitEnvVar("HUGO=rocks") c.Assert(key, qt.Equals, "HUGO") c.Assert(val, qt.Equals, "rocks") + + c.Assert(stringToGibabyte("1.0"), qt.Equals, uint64(1073741824)) } diff --git a/deps/deps.go b/deps/deps.go index f6b64c279e3..f15ab9a3436 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -5,6 +5,8 @@ import ( "sync/atomic" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/pkg/errors" "github.com/gohugoio/hugo/cache/filecache" @@ -63,9 +65,12 @@ type Deps struct { // The configuration to use Cfg config.Provider `json:"-"` - // The file cache to use. + // The file caches to use. FileCaches filecache.Caches + // The memory cache to use. + MemCache *memcache.Cache + // The translation func to use Translate func(translationID string, templateData interface{}) string `json:"-"` @@ -162,6 +167,13 @@ type ResourceProvider interface { Clone(deps *Deps) error } +// Stop stops all running caches etc. +func (d *Deps) Stop() { + if d.MemCache != nil { + d.MemCache.Stop() + } +} + func (d *Deps) Tmpl() tpl.TemplateHandler { return d.tmpl } @@ -240,11 +252,12 @@ func New(cfg DepsCfg) (*Deps, error) { if err != nil { return nil, errors.WithMessage(err, "failed to create file caches from configuration") } + memCache := memcache.New(memcache.Config{Running: cfg.Running}) errorHandler := &globalErrHandler{} buildState := &BuildState{} - resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) + resourceSpec, err := resources.NewSpec(ps, fileCaches, memCache, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -284,6 +297,7 @@ func New(cfg DepsCfg) (*Deps, error) { Language: cfg.Language, Site: cfg.Site, FileCaches: fileCaches, + MemCache: memCache, BuildStartListeners: &Listeners{}, BuildState: buildState, Running: cfg.Running, @@ -320,7 +334,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache postBuildAssets := d.ResourceSpec.PostBuildAssets - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) + d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.MemCache, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index fda7e2327a5..7d77551ab0d 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -375,9 +375,18 @@ Set `titleCaseStyle` to specify the title style used by the [title](/functions/t ## Configuration Environment Variables +### Control Parallel Processing + HUGO_NUMWORKERMULTIPLIER : Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used. +### Control Memory Usage + +HUGO_MEMORYLIMIT {{< new-in "0.77.0" >}} +: A value in Gigabytes (e.g. "1.5") to control the upper memory limit for Hugo's memory use. If not set, it will set aside a quarter of the total system memory. +Note that this does not represent a hard promise that Hugo will not use more memory than configured, but we will try our best. You may need to tune this setting depending on your site and/or system. + + ## Configuration Lookup Order Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior: diff --git a/go.mod b/go.mod index d927a646fa6..c0b99de9bd8 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/google/go-cmp v0.5.2 github.com/gorilla/websocket v1.4.2 github.com/jdkato/prose v1.2.0 + github.com/karlseguin/ccache/v2 v2.0.7-0.20200816131247-1189f7f993b5 github.com/kylelemons/godebug v1.1.0 github.com/kyokomi/emoji v2.2.4+incompatible github.com/magefile/mage v1.10.0 @@ -39,6 +40,7 @@ require ( github.com/nicksnyder/go-i18n/v2 v2.1.1 github.com/niklasfasching/go-org v1.3.2 github.com/olekukonko/tablewriter v0.0.4 + github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 github.com/pelletier/go-toml v1.8.1 github.com/pkg/errors v0.9.1 github.com/rogpeppe/go-internal v1.6.2 diff --git a/go.sum b/go.sum index cae36dad640..3395e9e62cd 100644 --- a/go.sum +++ b/go.sum @@ -7,16 +7,11 @@ cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISt cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= @@ -58,6 +53,8 @@ github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkK github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U= github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= github.com/alecthomas/chroma v0.7.2-0.20200305040604-4f3623dce67a/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s= +github.com/alecthomas/chroma v0.8.0 h1:HS+HE97sgcqjQGu5uVr8jIE55Mmh5UeQ7kckAhHg2pY= +github.com/alecthomas/chroma v0.8.0/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= github.com/alecthomas/chroma v0.8.1 h1:ym20sbvyC6RXz45u4qDglcgr8E313oPROshcuCHqiEE= github.com/alecthomas/chroma v0.8.1/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo= @@ -82,7 +79,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.19.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.35.0 h1:Pxqn1MWNfBCNcX7jrXCCTfsKpg5ms2IMUMmmcGtYJuo= +github.com/aws/aws-sdk-go v1.27.1 h1:MXnqY6SlWySaZAqNnXThOvjRFdiiOuKtC6i7baFdNdU= +github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.35.0/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -90,7 +88,8 @@ github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bep/gitmap v1.1.2 h1:zk04w1qc1COTZPPYWDQHvns3y1afOsdRfraFQ3qI840= github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= -github.com/bep/golibsass v0.7.0 h1:/ocxgtPZ5rgp7FA+mktzyent+fAg82tJq4iMsTMBAtA= +github.com/bep/golibsass v0.6.0 h1:WqJ8XC0Ri2210omWKwVVeaston02XhhArblb0ly6d6Y= +github.com/bep/golibsass v0.6.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= github.com/bep/golibsass v0.7.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= @@ -109,25 +108,21 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/disintegration/gift v1.2.1 h1:Y005a1X4Z7Uc+0gLpSAsKhWi4qLtsdEcMIbbdvdZ6pc= github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dUg3/RlKGr4HI= -github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg= github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -135,17 +130,13 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/evanw/esbuild v0.8.4 h1:FiqXH94T4y8/4MY3SLmPH1VjykKTfHMqtWYgxINOYxQ= github.com/evanw/esbuild v0.8.4/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw+Q= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.4.1 h1:Wv2VwvNn73pAdFIVUQRXYDFp31lXKbqblIXo/Q5GPSg= github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/frankban/quicktest v1.11.2 h1:mjwHjStlXWibxOohM7HYieIViKyh56mmt3+6viyhDDI= github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -168,48 +159,33 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 h1:sgew0XCnZwnzpWxTt3V8LLiCO7OQi3C6dycaE67wfkU= github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95/go.mod h1:bOlVlCa1/RajcHpXkrUXPSHB/Re1UnlXxD1Qp8SKOd8= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.2.2 h1:fSIRzE/K12IaNgV6X0173X/oLrTwHKRiMcFZhiDrN3s= github.com/google/wire v0.2.2/go.mod h1:7FHVg6mFpFQrjeUZrm+BaD50N5jnDKm50uVPTpyYOmU= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= @@ -217,7 +193,6 @@ github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -236,9 +211,7 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -247,55 +220,47 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jdkato/prose v1.2.0 h1:t/R3H6xOrVuIgNevWiOSJf1kEoeF2VWlrN6w76Tkzow= github.com/jdkato/prose v1.2.0/go.mod h1:WC4YKHtBdAMgBdmfdqBmEuVbBD0U5c9HQ6l1U8Cq0ts= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200816131247-1189f7f993b5 h1:oiSLAVaELZi+gW0jic4sNiI9cRq0/QdkDllDU15/aeA= +github.com/karlseguin/ccache/v2 v2.0.7-0.20200816131247-1189f7f993b5/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyokomi/emoji v2.2.4+incompatible h1:np0woGKwx9LiHAQmwZx79Oc0rHpNw3o+3evou4BEPv4= github.com/kyokomi/emoji v2.2.4+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c= github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88= github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -307,7 +272,6 @@ github.com/miekg/mmark v1.3.6 h1:t47x5vThdwgLJzofNsbsAl7gmIiJ7kbDQN5BxwBmwvY= github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -315,7 +279,6 @@ github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -325,7 +288,6 @@ github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFW github.com/muesli/smartcrop v0.3.0 h1:JTlSkmxWg/oQ1TcLDoypuirdE8Y/jzNirQeLkxpA6Oc= github.com/muesli/smartcrop v0.3.0/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nicksnyder/go-i18n/v2 v2.1.1 h1:ATCOanRDlrfKVB4WHAdJnLEqZtDmKYsweqsOUYflnBU= github.com/nicksnyder/go-i18n/v2 v2.1.1/go.mod h1:d++QJC9ZVf7pa48qrsRWhMJ5pSHIPmS3OLqK1niyLxs= @@ -341,20 +303,18 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 h1:MfIUBZ1bz7TgvQLVa/yPJZOGeKEgs6eTKUjz3zB4B+U= +github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -377,7 +337,6 @@ github.com/rogpeppe/go-internal v1.6.2 h1:aIihoIOHCiLZHxyoNQ+ABL4NKhFTgKLBdMLyEA github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6 h1:tlXG832s5pa9x9Gs3Rp2rTvEqjiDEuETUOSfBEiTcns= github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -385,53 +344,38 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sanity-io/litter v1.3.0 h1:5ZO+weUsqdSWMUng5JnpkW/Oz8iTXiIdeumhQr1sSjs= github.com/sanity-io/litter v1.3.0/go.mod h1:5Z71SvaYy5kcGtyglXOC9rrUi3c1E8CamFWjQsazTh0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shogo82148/go-shuffle v0.0.0-20180218125048-27e6095f230d/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/fsync v0.9.0 h1:f9CEt3DOB2mnHxZaftmEOFWjABEvKM/xpf3cUwJrGOY= github.com/spf13/fsync v0.9.0/go.mod h1:fNtJEfG3HiltN3y4cPOz6MLjos9+2pIEqLIgszqhp/0= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -439,7 +383,6 @@ github.com/tdewolff/minify/v2 v2.6.2 h1:Jaod6aSABWmhftvnxvXogxcEoQt6yogfFeZgIQEM github.com/tdewolff/minify/v2 v2.6.2/go.mod h1:BkDSm8aMMT0ALGmpt7j3Ra7nLUgZL0qhyrAHXwxcy5w= github.com/tdewolff/parse/v2 v2.4.2 h1:Bu2Qv6wepkc+Ou7iB/qHjAhEImlAP5vedzlQRUdj3BI= github.com/tdewolff/parse/v2 v2.4.2/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= -github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4= github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -449,10 +392,10 @@ github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/Aaua github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.22 h1:0e0f6Zee9SAQ5yOZGNMWaOxqVvcc/9/kUWu/Kl91Jk8= github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -463,16 +406,12 @@ go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -gocloud.dev v0.15.0 h1:Tl8dkOHWVZiYBYPxG2ouhpfmluoQGt3mY323DaAHaC8= gocloud.dev v0.15.0/go.mod h1:ShXCyJaGrJu9y/7a6+DSCyBb9MFGZ1P5wwPa0Wu6w34= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -486,7 +425,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -498,7 +436,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -506,7 +443,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -528,19 +464,14 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -549,7 +480,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U= golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -566,23 +496,17 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w= golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -602,7 +526,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -611,21 +534,16 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.5.0 h1:lj9SyhMzyoa38fgFF0oO2T6pjs5IzkLPKfVtxpyCRMM= google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -635,25 +553,19 @@ google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -662,20 +574,16 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= pack.ag/amqp v0.8.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= diff --git a/helpers/general.go b/helpers/general.go index 80e303087a5..377cba65578 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -61,6 +61,24 @@ func FindAvailablePort() (*net.TCPAddr, error) { return nil, err } +// FormatByteCount pretty formats b. +func FormatByteCount(bc uint64) string { + const ( + Gigabyte = 1 << 30 + Megabyte = 1 << 20 + Kilobyte = 1 << 10 + ) + switch { + case bc > Gigabyte || -bc > Gigabyte: + return fmt.Sprintf("%.2f GB", float64(bc)/Gigabyte) + case bc > Megabyte || -bc > Megabyte: + return fmt.Sprintf("%.2f MB", float64(bc)/Megabyte) + case bc > Kilobyte || -bc > Kilobyte: + return fmt.Sprintf("%.2f KB", float64(bc)/Kilobyte) + } + return fmt.Sprintf("%d B", bc) +} + // InStringArray checks if a string is an element of a slice of strings // and returns a boolean value. func InStringArray(arr []string, el string) bool { diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 5bfb1ff3233..141d87638af 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -41,6 +41,7 @@ const ( metaKeyBaseDir = "baseDir" // Abs base directory of source file. metaKeyMountRoot = "mountRoot" metaKeyModule = "module" + metaKeyComponent = "component" metaKeyOriginalFilename = "originalFilename" metaKeyName = "name" metaKeyPath = "path" @@ -137,6 +138,10 @@ func (f FileMeta) MountRoot() string { return f.stringV(metaKeyMountRoot) } +func (f FileMeta) Component() string { + return f.stringV(metaKeyComponent) +} + func (f FileMeta) Module() string { return f.stringV(metaKeyModule) } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index a38560d0a1b..38e526600f1 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -64,6 +64,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rm.Meta[metaKeyBaseDir] = rm.ToBasedir rm.Meta[metaKeyMountRoot] = rm.path rm.Meta[metaKeyModule] = rm.Module + rm.Meta[metaKeyComponent] = fromBase meta := copyFileMeta(rm.Meta) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index e79228ba340..70840b2d46e 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -251,6 +251,13 @@ func (m *pageMap) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resour } target := strings.TrimPrefix(meta.Path(), owner.File().Dir()) + tbase := meta.TranslationBaseNameWithExt() + if tbase != "" { + dir, base := filepath.Split(target) + if base != tbase { + target = filepath.Join(dir, tbase) + } + } return owner.s.ResourceSpec.New( resources.ResourceSourceDescriptor{ diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 189aa19c6cc..eb769a7eb6d 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -20,9 +20,12 @@ import ( "os" "path" "path/filepath" + "sort" "strings" "sync" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/hugofs/files" @@ -258,6 +261,21 @@ func (s SourceFilesystems) IsAsset(filename string) bool { return s.Assets.Contains(filename) } +// CollectPathIdentities collects paths relative to their component root from +// the data, content and assets filesystems. +func (s SourceFilesystems) CollectPathIdentities(filename string) identity.PathIdentities { + var identities identity.PathIdentities + + for _, fs := range []*SourceFilesystem{s.Assets, s.Content, s.Data} { + fs.withEachRelativePath(filename, func(rel string, meta hugofs.FileMeta) { + identities = append(identities, identity.NewPathIdentity(fs.Name, rel, meta.Filename(), meta.Lang())) + }) + + } + return identities + +} + // IsI18n returns true if the given filename is a member of the i18n filesystem. func (s SourceFilesystems) IsI18n(filename string) bool { return s.I18n.Contains(filename) @@ -277,21 +295,46 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { // MakePathRelative creates a relative path from the given filename. // It will return an empty string if the filename is not a member of this filesystem. +// TODO(bep) there may be multiple, see https://github.com/gohugoio/hugo/issues/7563 +// See collectRelativePaths func (d *SourceFilesystem) MakePathRelative(filename string) string { + paths := d.collectRelativePaths(filename) + if paths == nil { + return "" + } + return paths[0] + +} + +func (d *SourceFilesystem) collectRelativePaths(filename string) []string { + var paths []string + d.withEachRelativePath(filename, func(rel string, meta hugofs.FileMeta) { + paths = append(paths, rel) + }) + + return paths +} +func (d *SourceFilesystem) withEachRelativePath(filename string, cb func(rel string, meta hugofs.FileMeta)) { for _, dir := range d.Dirs { meta := dir.(hugofs.FileMetaInfo).Meta() currentPath := meta.Filename() - - if strings.HasPrefix(filename, currentPath) { - rel := strings.TrimPrefix(filename, currentPath) + if rel := relFilename(currentPath, filename); rel != "" { if mp := meta.Path(); mp != "" { - rel = filepath.Join(mp, rel) + cb(rel, meta) } - return strings.TrimPrefix(rel, filePathSeparator) } } - return "" +} + +func relFilename(dirname, filename string) string { + if !strings.HasSuffix(dirname, filePathSeparator) { + dirname += filePathSeparator + } + if !strings.HasPrefix(filename, dirname) { + return "" + } + return strings.TrimPrefix(filename, dirname) } func (d *SourceFilesystem) RealFilename(rel string) string { @@ -770,3 +813,21 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollect return b.createOverlayFs(collector, mounts[1:]) } + +func uniqueStringsSorted(s []string) []string { + if len(s) == 0 { + return nil + } + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 25ae3dd196f..c906bb57682 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -15,6 +15,7 @@ package hugolib import ( "context" + "fmt" "io" "path/filepath" "sort" @@ -699,8 +700,14 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool { return true } - if cfg.whatChanged != nil && !p.File().IsZero() { - return cfg.whatChanged.files[p.File().Filename()] + if cfg.whatChanged != nil { + pid := p.GetIdentity() + for _, p := range cfg.whatChanged.files { + if p.ProbablyEq(pid) { + return true + } + + } } return false @@ -972,7 +979,11 @@ func (h *HugoSites) resetPageState() { }) } -func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { +func (h *HugoSites) resetPageStateFromEvents(ids identity.PathIdentities) { + if ids == nil { + return + } + fmt.Println("====>", ids) h.getContentMaps().walkBundles(func(n *contentNode) bool { if n.p == nil { return false @@ -983,8 +994,8 @@ func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { if po.cp == nil { continue } - for id := range idset { - if po.cp.dependencyTracker.Search(id) != nil { + for _, id := range ids { + if po.cp.dependencyTracker.SearchProbablyDependent(id) != nil { po.cp.Reset() continue OUTPUTS } @@ -998,8 +1009,8 @@ func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { for _, s := range p.shortcodeState.shortcodes { for _, templ := range s.templs { sid := templ.(identity.Manager) - for id := range idset { - if sid.Search(id) != nil { + for _, id := range ids { + if sid.SearchProbablyDependent(id) != nil { for _, po := range p.pageOutputs { if po.cp != nil { po.cp.Reset() diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index bd5c2b6619f..97916e091ec 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -49,6 +49,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Make sure we don't trigger rebuilds in parallel. h.runningMu.Lock() defer h.runningMu.Unlock() + } else { + defer func() { + h.Stop() + }() } ctx, task := trace.NewTask(context.Background(), "Build") diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 8d0872bd5e0..3fee69bf3de 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -2,6 +2,7 @@ package hugolib import ( "fmt" + "os" "strings" "testing" @@ -11,6 +12,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource_transformers/tocss/scss" "github.com/fortytw2/leaktest" "github.com/fsnotify/fsnotify" @@ -417,15 +419,18 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { func TestMultiSitesRebuild(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 - // This leaktest seems to be a little bit shaky on Travis. - if !isCI() { - defer leaktest.CheckTimeout(t, 10*time.Second)() - } c := qt.New(t) b := newMultiSiteTestDefaultBuilder(t).Running().CreateSites().Build(BuildCfg{}) + defer b.H.Stop() + + // This leaktest seems to be a little bit shaky on Travis. + if !isCI() { + defer leaktest.CheckTimeout(t, 10*time.Second)() + } + sites := b.H.Sites fs := b.Fs @@ -1478,3 +1483,113 @@ func TestRebuildOnAssetChange(t *testing.T) { b.Build(BuildCfg{}) b.AssertFileContent("public/index.html", `changed data`) } + +func TestCacheOnRebuild(t *testing.T) { + c := qt.New(t) + + const jsonSample = `{ + "employee": { + "name": "Hugo", + "salary": 56000, + "married": false + } +}` + + createSitesBuilder := func(t testing.TB, prepare func(b *sitesBuilder)) *sitesBuilder { + b := newTestSitesBuilder(t).Running() + prepare(b) + b.Build(BuildCfg{}) + return b + + } + + c.Run("Page Resource minified", func(c *qt.C) { + + b := createSitesBuilder(c, func(b *sitesBuilder) { + b.WithContent("mysection/mybundle/index.md", "---title: MyBundle\n---", + "mysection/mybundle/data.json", jsonSample) + + b.WithTemplatesAdded("index.html", ` +{{ $p := site.GetPage "mysection/mybundle" }} +{{ $json := $p.Resources.GetMatch "data.json" }} +Unminified JSON: {{ $json.Content }} +{{ $min := $json | minify | fingerprint }} +Minified JSON: {{ $json.Content }} +JSON: {{ $min.RelPermalink }}|{{ $min.Content }} + +`) + + }) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +JSON: /mysection/mybundle/data.min.6394a5ec9f369469f6af29777270e5c6d73001969af6d2cb9c8ad6137e1bb00b.json| +Hugo`) + + newJSON := strings.ReplaceAll(jsonSample, "Hugo", "Fred") + b.EditFiles("content/mysection/mybundle/data.json", newJSON) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `AAA +Fred`) + + }) + + c.Run("transform.Unmarshal with temporary Resource", func(c *qt.C) { + templ := ` + {{ $m := "a: 1" | resources.FromString "a.yaml" | transform.Unmarshal }} + NM: {{ $m }} + + ` + b := createSitesBuilder(c, func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", templ) + }) + + b.AssertFileContent("public/index.html", `NM: map[a:1]`) + + templ = strings.ReplaceAll(templ, "a: 1", "a: 2") + templ = strings.ReplaceAll(templ, "NM:", "NM2:") + b.EditFiles("layouts/index.html", templ) + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `NM2: map[a:2]`) + + }) + + c.Run("SCSS with multiple roots", func(c *qt.C) { + if !scss.Supports() { + c.Skip("Skip SCSS") + } + + b, workDir, clean := newTestSitesBuilderWithOSFs(c, "scss-multiroot") + defer clean() + b.Running() + + assetsDir := filepath.Join(workDir, "assets") + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root1"), 0777), qt.IsNil) + c.Assert(os.MkdirAll(filepath.Join(assetsDir, "root2"), 0777), qt.IsNil) + + b.WithTemplatesAdded("index.html", ` +{{ $css := resources.Get "root2/main.scss" | toCSS }} +CSS: {{ $css.Content | safeHTML }} +`) + + b.WithSourceFile(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: blue;`) + b.WithSourceFile(filepath.Join(assetsDir, "root2/main.scss"), `@import "../root1/_vars"; + +body { color: $base-color; } +`) + + b.Build(BuildCfg{}) + b.AssertFileContent("public/index.html", `color: blue;`) + + b.EditFiles(filepath.Join(assetsDir, "root1/_vars.scss"), `$base-color: red;`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `color: red;`) + }) + +} diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 4fe4960422c..80885b5d67c 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -89,7 +89,7 @@ languageName = "Nynorsk" s2h := s2.getPage(page.KindHome) c.Assert(s2h.Permalink(), qt.Equals, "https://example.fr/") - b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /docs/text/pipes.txt") + b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /text/pipes.txt") b.AssertFileContent("public/fr/text/pipes.txt", "Hugo Pipes") b.AssertFileContent("public/en/index.html", "Default Home Page", "String Resource: /docs/text/pipes.txt") b.AssertFileContent("public/en/text/pipes.txt", "Hugo Pipes") diff --git a/hugolib/page.go b/hugolib/page.go index 859834b91e5..683c0f72cec 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -141,7 +141,19 @@ func (p *pageState) Eq(other interface{}) bool { } func (p *pageState) GetIdentity() identity.Identity { - return identity.NewPathIdentity(files.ComponentFolderContent, filepath.FromSlash(p.Path())) + var filename string + if !p.File().IsZero() { + filename = p.File().Filename() + } + return identity.NewPathIdentity(files.ComponentFolderContent, p.Path(), filename, p.Lang()) +} + +func (p *pageState) IsStale() bool { + return false +} + +func (l *pageState) IsNotDependent(other identity.Identity) bool { + panic("TODO1: pageState") } func (p *pageState) GitInfo() *gitmap.GitInfo { diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index fa420a025aa..5af55e738f0 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -347,7 +347,7 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { b.AssertFileContent("public/en/bc/data1.json", "data1") b.AssertFileContent("public/en/bc/data2.json", "data2") b.AssertFileContent("public/en/bc/logo-bc.png", "logo") - b.AssertFileContent("public/nn/bc/data1.nn.json", "data1.nn") + b.AssertFileContent("public/nn/bc/data1.json", "data1.nn") b.AssertFileContent("public/nn/bc/data2.json", "data2") b.AssertFileContent("public/nn/bc/logo-bc.png", "logo") @@ -1320,7 +1320,7 @@ bundle min min key: {{ $jsonMinMin.Key }} b.AssertFileContent(index, fmt.Sprintf("bundle min min min: /bundle%d/data.min.min.min.json", i), - fmt.Sprintf("bundle min min key: /bundle%d/data.min.min.json", i), + fmt.Sprintf("bundle min min key: bundle%d/content/en/data_c9a5f8c93697691d6f8a298931d285f7.json", i), ) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.min.min.json", i)), qt.Equals, true) b.Assert(b.CheckExists(fmt.Sprintf("public/bundle%d/data.min.json", i)), qt.Equals, false) diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 9f2e4bab137..346f3dd2d64 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -184,10 +184,6 @@ func (c *pagesCollector) Collect() (collectErr error) { } for dir := range dirs { - for _, pm := range c.contentMap.pmaps { - pm.s.ResourceSpec.DeleteBySubstring(dir.dirname) - } - switch dir.tp { case bundleLeaf: collectErr = c.collectDir(dir.dirname, true, nil) diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 97d4f17bab7..066c788fd16 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -53,6 +53,8 @@ type Paths struct { PublishDir string + IsMultiHost bool + // When in multihost mode, this returns a list of base paths below PublishDir // for each language. MultihostTargetBasePaths []string @@ -142,7 +144,8 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { } var multihostTargetBasePaths []string - if languages.IsMultihost() { + isMultiHost := languages.IsMultihost() + if isMultiHost { for _, l := range languages { multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang) } @@ -171,6 +174,7 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { Language: language, Languages: languages, LanguagesDefaultFirst: languagesDefaultFirst, + IsMultiHost: isMultiHost, MultihostTargetBasePaths: multihostTargetBasePaths, PaginatePath: cfg.GetString("paginatePath"), diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 7573199aacd..827bc04f5ae 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -355,7 +355,7 @@ Edited content. `) b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil) - b.H.ResourceSpec.ClearCaches() + b.H.MemCache.Clear() } } diff --git a/hugolib/resource_change_test.go b/hugolib/resource_change_test.go new file mode 100644 index 00000000000..941df8e7ea0 --- /dev/null +++ b/hugolib/resource_change_test.go @@ -0,0 +1,322 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "strings" + "testing" +) + +func TestResourceEditMetadata(t *testing.T) { + b := newTestSitesBuilder(t).Running() + + content := `+++ +title = "My Bundle With TOML Meta" + +[[resources]] +src = "**.toml" +title = "My TOML :counter" ++++ + +Content. +` + + b.WithContent( + "bundle/index.md", content, + "bundle/my1.toml", `a = 1`, + "bundle/my2.toml", `a = 2`) + + b.WithTemplatesAdded("index.html", ` +{{ $bundle := site.GetPage "bundle" }} +{{ $toml := $bundle.Resources.GetMatch "*.toml" }} +TOML: {{ $toml.Title }} + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My TOML 1") + + b.EditFiles("content/bundle/index.md", strings.ReplaceAll(content, "My TOML", "My Changed TOML 1")) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "TOML: My Changed TOML") + +} + +func TestResourceCacheSimpleTest(t *testing.T) { + conf := ` +baseURL = "https://example.org" + +defaultContentLanguage = "en" + +[module] +[[module.mounts]] +source = "content/cen" +target = "content" +lang="en" +[[module.mounts]] +source = "content/cno" +target = "content" +lang="no" +[[module.mounts]] +source = "assets" +target = "assets" +[[module.mounts]] +source = "assets_common" +target = "assets/aen" +[[module.mounts]] +source = "assets_common" +target = "assets/ano" + +[languages] +[languages.en] +weight = 1 + +[languages.no] +weight = 2 + +` + b := newTestSitesBuilder(t).WithConfigFile("toml", conf).Running() + + b.WithSourceFile( + "content/cen/bundle/index.md", "---\ntitle: En Bundle\n---", + "content/cen/bundle/data1.json", `{ "data1": "en" }`, + "content/cen/bundle/data2.json", `{ "data2": "en" }`, + "content/cno/bundle/index.md", "---\ntitle: No Bundle\n---", + "content/cno/bundle/data1.json", `{ "data1": "no" }`, + "content/cno/bundle/data3.json", `{ "data3": "no" }`, + ) + + b.WithSourceFile("assets_common/data/common.json", `{ + "Hugo": "Rocks!", + }`) + + b.WithSourceFile("assets/data/mydata.json", `{ + "a": 32, + }`) + + b.WithTemplatesAdded("index.html", ` +{{ $data := resources.Get "data/mydata.json" }} +{{ template "print-resource" ( dict "title" "data" "r" $data ) }} +{{ $dataMinified := $data | minify }} +{{ template "print-resource" ( dict "title" "data-minified" "r" $dataMinified ) }} +{{ $dataUnmarshaled := $dataMinified | transform.Unmarshal }} +Data Unmarshaled: {{ $dataUnmarshaled }} +{{ $bundle := site.GetPage "bundle" }} +{{ range (seq 3) }} +{{ $i := . }} +{{ with $bundle.Resources.GetMatch (printf "data%d.json" . ) }} +{{ $minified := . | minify }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d" $i) "r" . ) }} +{{ template "print-resource" ( dict "title" (printf "bundle data %d min" $i) "r" $minified ) }} +{{ end }} +{{ end }} +{{ $common1 := resources.Get "aen/data/common.json" }} +{{ $common2 := resources.Get "ano/data/common.json" }} +{{ template "print-resource" ( dict "title" "common1" "r" $common1 ) }} +{{ template "print-resource" ( dict "title" "common2" "r" $common2 ) }} +{{ define "print-resource" }}{{ .title }}|{{ .r.RelPermalink }}|{{ .r.Key }}|{{ .r.Content | safeHTML }}|{{ end }} + + + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +Data Unmarshaled: map[a:32] +bundle data 1|/bundle/data1.json|bundle/content/en/data1.json|{ "data1": "en" }| +bundle data 1 min|/bundle/data1.min.json|bundle/content/en/data1_d3f53f09220d597dac26fe7840c31fc9.json|{"data1":"en"}| +bundle data 3|/bundle/data3.json|bundle/content/en/data3.json|{ "data3": "no" }| +bundle data 3 min|/bundle/data3.min.json|bundle/content/en/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json| +common2|/ano/data/common.json|ano/data/common.json| +`) + + b.AssertFileContent("public/no/index.html", ` +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":32}| +bundle data 1|/no/bundle/data1.json|bundle/content/no/data1.json|{ "data1": "no" }| +bundle data 2|/no/bundle/data2.json|bundle/content/no/data2.json|{ "data2": "en" }| + bundle data 3|/no/bundle/data3.json|bundle/content/no/data3.json|{ "data3": "no" }| +bundle data 3 min|/no/bundle/data3.min.json|bundle/content/no/data3_d3f53f09220d597dac26fe7840c31fc9.json|{"data3":"no"}| +common1|/aen/data/common.json|aen/data/common.json +`) + + b.EditFiles("assets/data/mydata.json", `{ "a": 42 }`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +data|/data/mydata.json|data/mydata.json|{ "a": 42 }| +data-minified|/data/mydata.min.json|data/mydata_d3f53f09220d597dac26fe7840c31fc9.json|{"a":42}| +Data Unmarshaled: map[a:42] +`) + +} + +func TestResourceCacheMultihost(t *testing.T) { + toLang := func(format, lang string) string { + return fmt.Sprintf(format, lang) + } + + addContent := func(b *sitesBuilder, contentPath func(path, lang string) string) { + b.WithNoContentAdded() + for _, lang := range []string{"en", "fr"} { + b.WithSourceFile( + contentPath("b1/index.md", lang), toLang("---\ntitle: Bundle 1 %s\n---", lang), + contentPath("b1/styles/style11.css", lang), toLang(".%s1: { color: blue };", lang), + contentPath("b1/styles/style12.css", lang), toLang(".%s2: { color: red };", lang), + ) + b.WithSourceFile( + contentPath("b2/index.md", lang), toLang("---\ntitle: Bundle 2 %s\n---", lang), + contentPath("b2/styles/style21.css", lang), toLang(".%s21: { color: green };", lang), + contentPath("b2/styles/style22.css", lang), toLang(".%s22: { color: orange };", lang), + ) + } + } + + addTemplates := func(b *sitesBuilder) { + b.WithTemplates("_default/single.html", ` +{{ template "print-page" (dict "page" . "title" "Self") }} +{{ $other := site.Sites.First.GetPage "b1" }} +{{ template "print-page" (dict "page" $other "title" "Other") }} + + +{{ define "print-page" }} +{{ $p := .page }} +{{ $title := .title }} +{{ $styles := $p.Resources.Match "**/style1*.css" }} +{{ if $styles }} +{{ $firststyle := index $styles 0 }} +{{ $mystyles := $styles | resources.Concat "mystyles.css" }} +{{ $title }} Mystyles First CSS: {{ $firststyle.RelPermalink }}|Key: {{ $firststyle.Key }}|{{ $firststyle.Content }}| +{{ $title }} Mystyles CSS: {{ $mystyles.RelPermalink }}|Key: {{ $mystyles.Key }}|{{ $mystyles.Content }}| +{{ end }} +{{ $title }} Bundle: {{ $p.Permalink }} +{{ $style := $p.Resources.GetMatch "**.css" }} +{{ $title }} CSS: {{ $style.RelPermalink }}|Key: {{ $style.Key }}|{{ $style.Content }}| +{{ $minified := $style | minify }} +{{ $title }} Minified CSS: {{ $minified.RelPermalink }}|Key: {{ $minified.Key }}|{{ $minified.Content }} +{{ end }} + +`) + } + + assertContent := func(b *sitesBuilder) { + + otherAssert := `Other Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Other Bundle: https://example.com/b1/ +Other CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue }; +Other Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue}` + + b.AssertFileContent("public/fr/b1/index.html", ` +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red }; +Self Bundle: https://example.fr/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/fr/styles/style11.css|.fr1: { color: blue }; +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/fr/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.fr1:{color:blue} +`, + otherAssert) + b.AssertFileContent("public/en/b1/index.html", ` +Self Mystyles First CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Mystyles CSS: /mystyles.css|Key: _root/mystyles.css|.en1: { color: blue };.en2: { color: red };| +Self Bundle: https://example.com/b1/ +Self CSS: /b1/styles/style11.css|Key: b1/content/en/styles/style11.css|.en1: { color: blue };| +Self Minified CSS: /b1/styles/style11.min.css|Key: b1/content/en/styles/style11_d3f53f09220d597dac26fe7840c31fc9.css|.en1:{color:blue} +`, otherAssert) + + b.AssertFileContent("public/fr/b2/index.html", ` +Self Bundle: https://example.fr/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/fr/styles/style21.css|.fr21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/fr/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.fr21:{color:green} +`, otherAssert) + + b.AssertFileContent("public/en/b2/index.html", ` +Self Bundle: https://example.com/b2/ +Self CSS: /b2/styles/style21.css|Key: b2/content/en/styles/style21.css|.en21: { color: green };| +Self Minified CSS: /b2/styles/style21.min.css|Key: b2/content/en/styles/style21_d3f53f09220d597dac26fe7840c31fc9.css|.en21:{color:green} +`, otherAssert) + + } + + t.Run("Default content", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false +contentDir = "content" + +[Languages] +[Languages.en] +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + fmt.Println(b.workingDir) + addContent(b, func(path, lang string) string { + path = strings.Replace(path, ".", "."+lang+".", 1) + path = "content/" + path + return path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + + t.Run("Content dir per language", func(t *testing.T) { + + var configTemplate = ` +paginate = 1 +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = false + +[Languages] +[Languages.en] +contentDir = "content_en" +baseURL = "https://example.com/" +weight = 10 +languageName = "English" + +[Languages.fr] +contentDir = "content_fr" +baseURL = "https://example.fr" +weight = 20 +languageName = "Français" + +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", configTemplate) + addContent(b, func(path, lang string) string { + return "content_" + lang + "/" + path + }) + addTemplates(b) + b.Build(BuildCfg{}) + assertContent(b) + + }) + +} diff --git a/hugolib/site.go b/hugolib/site.go index 3679e354cca..f79cf77ce49 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -29,12 +29,12 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/constants" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/converter/hooks" @@ -915,7 +915,7 @@ func (s *Site) multilingual() *Multilingual { type whatChanged struct { source bool - files map[string]bool + files identity.PathIdentities } // RegisterMediaTypes will register the Site's media types in the mime @@ -1014,8 +1014,6 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro events = s.filterFileEvents(events) events = s.translateFileEvents(events) - changeIdentities := make(identity.Identities) - s.Log.Debug().Printf("Rebuild for events %q", events) h := s.h @@ -1023,75 +1021,79 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro // First we need to determine what changed var ( - sourceChanged = []fsnotify.Event{} sourceReallyChanged = []fsnotify.Event{} contentFilesChanged []string - tmplChanged bool - tmplAdded bool - dataChanged bool - i18nChanged bool - - sourceFilesChanged = make(map[string]bool) + tmplChanged bool + tmplAdded bool + dataChanged bool + i18nChanged bool + contentChanged bool // prevent spamming the log on changes logger = helpers.NewDistinctFeedbackLogger() ) - var cachePartitions []string - // Special case - // TODO(bep) I have a ongoing branch where I have redone the cache. Consider this there. - var ( - evictCSSRe *regexp.Regexp - evictJSRe *regexp.Regexp - ) - + // Paths relative to their component folder. + var resourcePaths identity.PathIdentities for _, ev := range events { - if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { - cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) - if evictCSSRe == nil { - if cssFileRe.MatchString(assetsFilename) || cssConfigRe.MatchString(assetsFilename) { - evictCSSRe = cssFileRe - } - } - if evictJSRe == nil && jsFileRe.MatchString(assetsFilename) { - evictJSRe = jsFileRe + removed := false + + if ev.Op&fsnotify.Remove == fsnotify.Remove { + removed = true + } + + // Some editors (Vim) sometimes issue only a Rename operation when writing an existing file + // Sometimes a rename operation means that file has been renamed other times it means + // it's been updated + if ev.Op&fsnotify.Rename == fsnotify.Rename { + // If the file is still on disk, it's only been updated, if it's not, it's been moved + if ex, err := afero.Exists(s.Fs.Source, ev.Name); !ex || err != nil { + removed = true } } - id, found := s.eventToIdentity(ev) - if found { - changeIdentities[id] = id - - switch id.Type { - case files.ComponentFolderContent: - logger.Println("Source changed", ev) - sourceChanged = append(sourceChanged, ev) - case files.ComponentFolderLayouts: - tmplChanged = true - if !s.Tmpl().HasTemplate(id.Path) { - tmplAdded = true - } - if tmplAdded { - logger.Println("Template added", ev) - } else { - logger.Println("Template changed", ev) - } + if removed && files.IsContentFile(ev.Name) { + h.removePageByFilename(ev.Name) + } - case files.ComponentFolderData: - logger.Println("Data changed", ev) - dataChanged = true - case files.ComponentFolderI18n: - logger.Println("i18n changed", ev) - i18nChanged = true + if paths := s.BaseFs.CollectPathIdentities(ev.Name); paths != nil { + resourcePaths = append(resourcePaths, paths...) + } + } + + fmt.Println(">>> RE", resourcePaths) + for _, id := range resourcePaths { + switch id.Type() { + case files.ComponentFolderContent: + logger.Println("Source changed", id.Filename()) + case files.ComponentFolderLayouts: + tmplChanged = true + if !s.Tmpl().HasTemplate(id.Path()) { + tmplAdded = true } + if tmplAdded { + logger.Println("Template added", id.Filename()) + } else { + logger.Println("Template changed", id.Filename()) + } + + case files.ComponentFolderData: + logger.Println("Data changed", id.Filename()) + dataChanged = true + case files.ComponentFolderI18n: + logger.Println("i18n changed", id.Filename()) + i18nChanged = true } + } + contentFiles := resourcePaths.ByType(files.ComponentFolderContent) + changed := &whatChanged{ - source: len(sourceChanged) > 0, - files: sourceFilesChanged, + source: contentChanged, + files: contentFiles, } config.whatChanged = changed @@ -1100,16 +1102,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro return err } - // These in memory resource caches will be rebuilt on demand. - for _, s := range s.h.Sites { - s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) - if evictCSSRe != nil { - s.ResourceSpec.ResourceCache.DeleteMatches(evictCSSRe) - } - if evictJSRe != nil { - s.ResourceSpec.ResourceCache.DeleteMatches(evictJSRe) - } - } + h.MemCache.ClearOn(memcache.ClearOnRebuild, resourcePaths...) if tmplChanged || i18nChanged { sites := s.h.Sites @@ -1144,35 +1137,10 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro s.h.init.data.Reset() } - for _, ev := range sourceChanged { - removed := false - - if ev.Op&fsnotify.Remove == fsnotify.Remove { - removed = true - } - - // Some editors (Vim) sometimes issue only a Rename operation when writing an existing file - // Sometimes a rename operation means that file has been renamed other times it means - // it's been updated - if ev.Op&fsnotify.Rename == fsnotify.Rename { - // If the file is still on disk, it's only been updated, if it's not, it's been moved - if ex, err := afero.Exists(s.Fs.Source, ev.Name); !ex || err != nil { - removed = true - } - } - - if removed && files.IsContentFile(ev.Name) { - h.removePageByFilename(ev.Name) - } - - sourceReallyChanged = append(sourceReallyChanged, ev) - sourceFilesChanged[ev.Name] = true - } - if config.ErrRecovery || tmplAdded || dataChanged { h.resetPageState() } else { - h.resetPageStateFromEvents(changeIdentities) + h.resetPageStateFromEvents(resourcePaths) } if len(sourceReallyChanged) > 0 || len(contentFilesChanged) > 0 { @@ -1364,13 +1332,14 @@ func (s *Site) initializeSiteInfo() error { return nil } +// TODO1 remove func (s *Site) eventToIdentity(e fsnotify.Event) (identity.PathIdentity, bool) { for _, fs := range s.BaseFs.SourceFilesystems.FileSystems() { if p := fs.Path(e.Name); p != "" { - return identity.NewPathIdentity(fs.Name, filepath.ToSlash(p)), true + return identity.NewPathIdentity(fs.Name, filepath.ToSlash(p), "", ""), true } } - return identity.PathIdentity{}, false + return nil, false } func (s *Site) readAndProcessContent(filenames ...string) error { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 5b825cd1e1c..bc6df1c3da1 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -92,7 +92,8 @@ type sitesBuilder struct { // Consider this in relation to using the BaseFs.PublishFs to all publishing. workingDir string - addNothing bool + addNothing bool + addNoContent bool // Base data/content contentFilePairs []filenameContent templateFilePairs []filenameContent @@ -126,6 +127,21 @@ func newTestSitesBuilder(t testing.TB) *sitesBuilder { dumper: litterOptions, rnd: rand.New(rand.NewSource(time.Now().Unix()))} } +func newTestSitesBuilderWithOSFs(t testing.TB, testname string) (*sitesBuilder, string, func()) { + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, testname) + if err != nil { + t.Fatal(err) + } + v := viper.New() + v.Set("workingDir", workDir) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) + + return b, workDir, clean +} + func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder { c := qt.New(t) @@ -154,6 +170,11 @@ func (s *sitesBuilder) WithNothingAdded() *sitesBuilder { return s } +func (s *sitesBuilder) WithNoContentAdded() *sitesBuilder { + s.addNoContent = true + return s +} + func (s *sitesBuilder) WithLogger(logger loggers.Logger) *sitesBuilder { s.logger = logger return s @@ -502,6 +523,7 @@ func (s *sitesBuilder) LoadConfig() error { } func (s *sitesBuilder) CreateSitesE() error { + if !s.addNothing { if _, ok := s.Fs.Source.(*afero.OsFs); ok { for _, dir := range []string{ @@ -529,7 +551,6 @@ func (s *sitesBuilder) CreateSitesE() error { s.writeFilePairs("data", s.dataFilePairs) s.writeFilePairs("content", s.contentFilePairs) s.writeFilePairs("layouts", s.templateFilePairs) - } if err := s.LoadConfig(); err != nil { @@ -677,7 +698,7 @@ hello: } ) - if len(s.contentFilePairs) == 0 { + if len(s.contentFilePairs) == 0 && !s.addNoContent { s.writeFilePairs("content", s.createFilenameContent(defaultContent)) } diff --git a/identity/identity.go b/identity/identity.go index 8fce16479bb..59ddddf9b52 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -1,10 +1,24 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package provides ways to identify values in Hugo. Used for dependency tracking etc. package identity import ( - "path/filepath" - "strings" "sync" "sync/atomic" + + "github.com/gohugoio/hugo/compare" ) // NewIdentityManager creates a new Manager starting at id. @@ -15,17 +29,10 @@ func NewManager(id Provider) Manager { } } -// NewPathIdentity creates a new Identity with the two identifiers -// type and path. -func NewPathIdentity(typ, pat string) PathIdentity { - pat = strings.ToLower(strings.TrimPrefix(filepath.ToSlash(pat), "/")) - return PathIdentity{Type: typ, Path: pat} -} - // Identities stores identity providers. type Identities map[Identity]Provider -func (ids Identities) search(depth int, id Identity) Provider { +func (ids Identities) search(depth int, probableMatch bool, id Identity) Provider { if v, found := ids[id.GetIdentity()]; found { return v @@ -40,9 +47,17 @@ func (ids Identities) search(depth int, id Identity) Provider { } for _, v := range ids { + if probableMatch { + if pid, ok := v.GetIdentity().(IsNotDependentProvider); ok { + if !pid.IsNotDependent(id.GetIdentity()) { + return v + } + } + } + switch t := v.(type) { case IdentitiesProvider: - if nested := t.GetIdentities().search(depth, id); nested != nil { + if nested := t.GetIdentities().search(depth, probableMatch, id); nested != nil { return nested } } @@ -59,6 +74,9 @@ type IdentitiesProvider interface { // any Go type, but the Identity returned by GetIdentify must be hashable. type Identity interface { Provider + compare.ProbablyEqer + compare.Eqer + Name() string } @@ -68,25 +86,10 @@ type Manager interface { Provider Add(ids ...Provider) Search(id Identity) Provider + SearchProbablyDependent(id Identity) Provider Reset() } -// A PathIdentity is a common identity identified by a type and a path, e.g. "layouts" and "_default/single.html". -type PathIdentity struct { - Type string - Path string -} - -// GetIdentity returns itself. -func (id PathIdentity) GetIdentity() Identity { - return id -} - -// Name returns the Path. -func (id PathIdentity) Name() string { - return id.Path -} - // A KeyValueIdentity a general purpose identity. type KeyValueIdentity struct { Key string @@ -103,11 +106,25 @@ func (id KeyValueIdentity) Name() string { return id.Key } +func (id KeyValueIdentity) Eq(other interface{}) bool { + return id == other +} + +func (id KeyValueIdentity) ProbablyEq(other interface{}) bool { + return id == other +} + // Provider provides the hashable Identity. type Provider interface { GetIdentity() Identity } +// IsNotDependentProvider provides a method to determin if the other +// Identity is not dependent on this one. +type IsNotDependentProvider interface { + IsNotDependent(other Identity) bool +} + type identityManager struct { sync.Mutex Provider @@ -140,7 +157,13 @@ func (im *identityManager) GetIdentities() Identities { func (im *identityManager) Search(id Identity) Provider { im.Lock() defer im.Unlock() - return im.ids.search(0, id.GetIdentity()) + return im.ids.search(0, false, id.GetIdentity()) +} + +func (im *identityManager) SearchProbablyDependent(id Identity) Provider { + im.Lock() + defer im.Unlock() + return im.ids.search(0, true, id.GetIdentity()) } // Incrementer increments and returns the value. diff --git a/identity/identity_test.go b/identity/identity_test.go index c315df1e720..0edf6f83991 100644 --- a/identity/identity_test.go +++ b/identity/identity_test.go @@ -32,6 +32,18 @@ func TestIdentityManager(t *testing.T) { c.Assert(im.Search(testIdentity{name: "notfound"}), qt.Equals, nil) } +func TestKeyValueIdentity(t *testing.T) { + c := qt.New(t) + + c.Assert(KeyValueIdentity{Key: "a", Value: "b"}.Eq(KeyValueIdentity{Key: "a", Value: "b"}), qt.Equals, true) + c.Assert(KeyValueIdentity{Key: "a", Value: "b"}.Eq(KeyValueIdentity{Key: "a", Value: "c"}), qt.Equals, false) + c.Assert(KeyValueIdentity{Key: "a", Value: "b"}.Eq(KeyValueIdentity{Key: "c", Value: "b"}), qt.Equals, false) + + c.Assert(KeyValueIdentity{Key: "a", Value: "b"}.ProbablyEq(KeyValueIdentity{Key: "a", Value: "b"}), qt.Equals, true) + c.Assert(KeyValueIdentity{Key: "a", Value: "b"}.ProbablyEq(KeyValueIdentity{Key: "a", Value: "c"}), qt.Equals, false) + c.Assert(KeyValueIdentity{Key: "a", Value: "b"}.ProbablyEq(KeyValueIdentity{Key: "c", Value: "b"}), qt.Equals, false) +} + func BenchmarkIdentityManager(b *testing.B) { createIds := func(num int) []Identity { @@ -91,3 +103,10 @@ func (id testIdentity) GetIdentity() Identity { func (id testIdentity) Name() string { return id.name } + +func (id testIdentity) Eq(other interface{}) bool { + return false +} +func (id testIdentity) ProbablyEq(other interface{}) bool { + return false +} diff --git a/identity/pathIdentity.go b/identity/pathIdentity.go new file mode 100644 index 00000000000..15ef2b9b37f --- /dev/null +++ b/identity/pathIdentity.go @@ -0,0 +1,167 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package identity + +import ( + "path/filepath" + "sort" + "strings" + + "github.com/gohugoio/hugo/hugofs/files" +) + +// NewPathIdentity creates a new Identity with the three identifiers +// type, path and lang (optional). +func NewPathIdentity(typ, path, filename, lang string) PathIdentity { + path = strings.ToLower(strings.TrimPrefix(filepath.ToSlash(path), "/")) + return pathIdentity{typ: typ, path: path, filename: filename, lang: lang} +} + +type PathIdentities []PathIdentity + +func (pp PathIdentities) ByType(typ string) PathIdentities { + var res PathIdentities + for _, p := range pp { + if p.Type() == typ { + res = append(res, p) + } + } + + return res + +} + +// UniqueSorted will remove duplicates and sort. The slice will be modified. +func (pp PathIdentities) UniqueSorted() PathIdentities { + set := map[PathIdentity]interface{}{} + result := pp[:0] + for _, val := range pp { + if _, ok := set[val]; !ok { + result = append(result, val) + set[val] = val + } + } + + sort.Slice(result, func(i, j int) bool { + pi, pj := result[i], result[j] + if pi.Path() != pj.Path() { + return pi.Path() < pj.Path() + } + + if pi.Filename() != pj.Filename() { + return pi.Filename() < pj.Filename() + } + + if pi.Lang() != pj.Lang() { + return pi.Lang() < pj.Lang() + } + + return pi.Type() < pj.Type() + + }) + return result +} + +// A PathIdentity is a common identity identified by a type and a path, +// e.g. "layouts" and "_default/single.html". +type PathIdentity interface { + Identity + Type() string + Path() string + Filename() string + Lang() string +} + +var _ IsNotDependentProvider = (*pathIdentity)(nil) + +type pathIdentity struct { + typ string + path string + filename string + lang string +} + +// GetIdentity returns itself. +func (id pathIdentity) GetIdentity() Identity { + return id +} + +func isCrossComponent(c string) bool { + return c == files.ComponentFolderData || c == files.ComponentFolderLayouts +} + +func (id pathIdentity) IsNotDependent(other Identity) bool { + pid, ok := other.GetIdentity().(PathIdentity) + if !ok { + return true + } + + if isCrossComponent(id.Type()) { + return id.Type() == pid.Type() + } + + if isCrossComponent(pid.Type()) { + // Changes in /data and /layouts currently triggers full + // content re-renders. + return id.Type() != files.ComponentFolderContent + } + + return true +} + +func (id pathIdentity) Type() string { + return id.typ +} + +func (id pathIdentity) Path() string { + return id.path +} + +func (id pathIdentity) Filename() string { + return id.filename +} + +func (id pathIdentity) Lang() string { + return id.lang +} + +// Name returns the Path. +func (id pathIdentity) Name() string { + return id.path +} + +func (id pathIdentity) Eq(other interface{}) bool { + return id == other +} + +func (id pathIdentity) ProbablyEq(other interface{}) bool { + if id == other { + return true + } + + oid, ok := other.(PathIdentity) + if !ok { + return false + } + + if id.Filename() != "" && oid.Filename() != "" && id.Filename() != oid.Filename() { + return false + } + + if id.Lang() == "" || oid.Lang() == "" && id.Lang() != oid.Lang() { + return id.Type() == oid.Type() && id.Path() == oid.Path() + } + + return false +} diff --git a/identity/pathIdentity_test.go b/identity/pathIdentity_test.go new file mode 100644 index 00000000000..347f6ce9871 --- /dev/null +++ b/identity/pathIdentity_test.go @@ -0,0 +1,36 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package identity + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestPathIdentity(t *testing.T) { + c := qt.New(t) + + c.Assert(pathIdentity{typ: "a", path: "b"}.Eq(pathIdentity{typ: "a", path: "b"}), qt.Equals, true) + c.Assert(pathIdentity{typ: "a", path: "b"}.Eq(pathIdentity{typ: "a", path: "c"}), qt.Equals, false) + c.Assert(pathIdentity{typ: "a", path: "b"}.Eq(pathIdentity{typ: "c", path: "b"}), qt.Equals, false) + + c.Assert(pathIdentity{typ: "a", path: "b"}.ProbablyEq(pathIdentity{typ: "a", path: "b"}), qt.Equals, true) + c.Assert(pathIdentity{typ: "a", path: "b", lang: "en"}.ProbablyEq(pathIdentity{typ: "a", path: "b"}), qt.Equals, true) + c.Assert(pathIdentity{typ: "a", path: "b", lang: "en"}.ProbablyEq(pathIdentity{typ: "a", path: "b", lang: "jp"}), qt.Equals, false) + + c.Assert(pathIdentity{typ: "a", path: "b"}.ProbablyEq(pathIdentity{typ: "a", path: "c"}), qt.Equals, false) + c.Assert(pathIdentity{typ: "a", path: "b"}.ProbablyEq(pathIdentity{typ: "c", path: "b"}), qt.Equals, false) + +} diff --git a/markup/converter/converter.go b/markup/converter/converter.go index 23152744193..36141a25175 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -131,5 +131,5 @@ type RenderContext struct { } var ( - FeatureRenderHooks = identity.NewPathIdentity("markup", "renderingHooks") + FeatureRenderHooks = identity.NewPathIdentity("markup", "renderingHooks", "", "") ) diff --git a/modules/client.go b/modules/client.go index 7d2175c94f3..834aedd2787 100644 --- a/modules/client.go +++ b/modules/client.go @@ -355,7 +355,7 @@ var verifyErrorDirRe = regexp.MustCompile(`dir has been modified \((.*?)\)`) // which are stored in a local downloaded source cache, have not been // modified since being downloaded. func (c *Client) Verify(clean bool) error { - // TODO1 add path to mod clean + // TODO(bep) add path to mod clean err := c.runVerify() if err != nil { diff --git a/resources/image.go b/resources/image.go index e999c5d96c2..78f390b4ec1 100644 --- a/resources/image.go +++ b/resources/image.go @@ -371,7 +371,7 @@ func (i *imageResource) getImageMetaCacheTargetPath() string { df.dir = filepath.Dir(fi.Meta().Path()) } p1, _ := helpers.FileAndExt(df.file) - h, _ := i.hash() + h := i.hash() idStr := helpers.HashString(h, i.size(), imageMetaVersionNumber, cfg) return path.Join(df.dir, fmt.Sprintf("%s_%s.json", p1, idStr)) } @@ -382,7 +382,7 @@ func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile p2 = conf.TargetFormat.DefaultExtension() } - h, _ := i.hash() + h := i.hash() idStr := fmt.Sprintf("_hu%s_%d", h, i.size()) // Do not change for no good reason. diff --git a/resources/image_cache.go b/resources/image_cache.go index 1888b457f59..3cc613c832d 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -17,8 +17,8 @@ import ( "image" "io" "path/filepath" - "strings" - "sync" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/images" @@ -30,36 +30,7 @@ type imageCache struct { pathSpec *helpers.PathSpec fileCache *filecache.Cache - - mu sync.RWMutex - store map[string]*resourceAdapter -} - -func (c *imageCache) deleteIfContains(s string) { - c.mu.Lock() - defer c.mu.Unlock() - s = c.normalizeKeyBase(s) - for k := range c.store { - if strings.Contains(k, s) { - delete(c.store, k) - } - } -} - -// The cache key is a lowecase path with Unix style slashes and it always starts with -// a leading slash. -func (c *imageCache) normalizeKey(key string) string { - return "/" + c.normalizeKeyBase(key) -} - -func (c *imageCache) normalizeKeyBase(key string) string { - return strings.Trim(strings.ToLower(filepath.ToSlash(key)), "/") -} - -func (c *imageCache) clear() { - c.mu.Lock() - defer c.mu.Unlock() - c.store = make(map[string]*resourceAdapter) + mCache memcache.Getter } func (c *imageCache) getOrCreate( @@ -67,101 +38,92 @@ func (c *imageCache) getOrCreate( createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) { relTarget := parent.relTargetPathFromConfig(conf) memKey := parent.relTargetPathForRel(relTarget.path(), false, false, false) - memKey = c.normalizeKey(memKey) + memKey = memcache.CleanKey(memKey) - // For the file cache we want to generate and store it once if possible. - fileKeyPath := relTarget - if fi := parent.root.getFileInfo(); fi != nil { - fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) - } - fileKey := fileKeyPath.path() + v, err := c.mCache.GetOrCreate(memKey, func() memcache.Entry { + // For the file cache we want to generate and store it once if possible. + fileKeyPath := relTarget + if fi := parent.root.getFileInfo(); fi != nil { + fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path())) + } + fileKey := fileKeyPath.path() - // First check the in-memory store, then the disk. - c.mu.RLock() - cachedImage, found := c.store[memKey] - c.mu.RUnlock() + var img *imageResource - if found { - return cachedImage, nil - } + // These funcs are protected by a named lock. + // read clones the parent to its new name and copies + // the content to the destinations. + read := func(info filecache.ItemInfo, r io.ReadSeeker) error { + img = parent.clone(nil) + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - var img *imageResource + if err := img.InitConfig(r); err != nil { + return err + } - // These funcs are protected by a named lock. - // read clones the parent to its new name and copies - // the content to the destinations. - read := func(info filecache.ItemInfo, r io.ReadSeeker) error { - img = parent.clone(nil) - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) + r.Seek(0, 0) - if err := img.InitConfig(r); err != nil { - return err - } + w, err := img.openDestinationsForWriting() + if err != nil { + return err + } - r.Seek(0, 0) + if w == nil { + // Nothing to write. + return nil + } + + defer w.Close() + _, err = io.Copy(w, r) - w, err := img.openDestinationsForWriting() - if err != nil { return err } - if w == nil { - // Nothing to write. - return nil - } + // create creates the image and encodes it to the cache (w). + create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + defer w.Close() - defer w.Close() - _, err = io.Copy(w, r) + var conv image.Image + img, conv, err = createImage() + if err != nil { + return + } + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - return err - } + return img.EncodeTo(conf, conv, w) + } + + // Now look in the file cache. - // create creates the image and encodes it to the cache (w). - create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { - defer w.Close() + // The definition of this counter is not that we have processed that amount + // (e.g. resized etc.), it can be fetched from file cache, + // but the count of processed image variations for this site. + c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - var conv image.Image - img, conv, err = createImage() + _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { - return + return memcache.Entry{Err: err} } - rp := img.getResourcePaths() - rp.relTargetDirFile.file = relTarget.file - img.setSourceFilename(info.Name) - return img.EncodeTo(conf, conv, w) - } + // The file is now stored in this cache. + img.setSourceFs(c.fileCache.Fs) - // Now look in the file cache. + imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - // The definition of this counter is not that we have processed that amount - // (e.g. resized etc.), it can be fetched from file cache, - // but the count of processed image variations for this site. - c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) + return memcache.Entry{Value: imgAdapter, ClearWhen: memcache.ClearOnChange} + }) - _, err := c.fileCache.ReadOrCreate(fileKey, read, create) if err != nil { return nil, err } + return v.(*resourceAdapter), nil - // The file is now stored in this cache. - img.setSourceFs(c.fileCache.Fs) - - c.mu.Lock() - if cachedImage, found = c.store[memKey]; found { - c.mu.Unlock() - return cachedImage, nil - } - - imgAdapter := newResourceAdapter(parent.getSpec(), true, img) - c.store[memKey] = imgAdapter - c.mu.Unlock() - - return imgAdapter, nil } -func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { - return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)} +func newImageCache(fileCache *filecache.Cache, memCache *memcache.Cache, ps *helpers.PathSpec) *imageCache { + return &imageCache{fileCache: fileCache, mCache: memCache.GetOrCreatePartition("images", memcache.ClearOnChange), pathSpec: ps} } diff --git a/resources/image_test.go b/resources/image_test.go index 1be9a5f8d0e..2d3020a6241 100644 --- a/resources/image_test.go +++ b/resources/image_test.go @@ -355,7 +355,7 @@ func TestImageResizeInSubPath(t *testing.T) { c.Assert(spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) // Cleare mem cache to simulate reading from the file cache. - spec.imageCache.clear() + spec.imageCache.mCache.Clear() resizedAgain, err := image.Resize("101x101") c.Assert(err, qt.IsNil) diff --git a/resources/page/page_nop.go b/resources/page/page_nop.go index 293b399c7bb..de28d3ba624 100644 --- a/resources/page/page_nop.go +++ b/resources/page/page_nop.go @@ -217,6 +217,14 @@ func (p *nopPage) IsDraft() bool { return false } +func (p *nopPage) IsStale() bool { + return false +} + +func (p *nopPage) IsNotDependent(other identity.Identity) bool { + return false +} + func (p *nopPage) IsHome() bool { return false } @@ -493,5 +501,5 @@ func (p *nopPage) WordCount() int { } func (p *nopPage) GetIdentity() identity.Identity { - return identity.NewPathIdentity("content", "foo/bar.md") + return identity.NewPathIdentity("content", "foo/bar.md", "", "t") } diff --git a/resources/page/pages.go b/resources/page/pages.go index ac69a8079f2..0b107c7bc50 100644 --- a/resources/page/pages.go +++ b/resources/page/pages.go @@ -25,6 +25,7 @@ import ( var ( _ resource.ResourcesConverter = Pages{} _ compare.ProbablyEqer = Pages{} + _ resource.StaleInfo = Pages{} ) // Pages is a slice of pages. This is the most common list type in Hugo. @@ -34,6 +35,16 @@ func (ps Pages) String() string { return fmt.Sprintf("Pages(%d)", len(ps)) } +// Pages is stale if any of the the pages are stale. +func (ps Pages) IsStale() bool { + for _, p := range ps { + if p.IsStale() { + return true + } + } + return false +} + // Used in tests. func (ps Pages) shuffle() { for i := range ps { diff --git a/resources/page/testhelpers_test.go b/resources/page/testhelpers_test.go index 54a908d3bf2..3efb22a03fc 100644 --- a/resources/page/testhelpers_test.go +++ b/resources/page/testhelpers_test.go @@ -265,6 +265,14 @@ func (p *testPage) IsDescendant(other interface{}) (bool, error) { panic("not implemented") } +func (p *testPage) IsStale() bool { + return false +} + +func (p *testPage) IsNotDependent(other identity.Identity) bool { + return false +} + func (p *testPage) IsDraft() bool { return false } diff --git a/resources/resource.go b/resources/resource.go index acdf2d744ec..56837e907ec 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -22,6 +22,9 @@ import ( "path/filepath" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/resources/internal" "github.com/gohugoio/hugo/common/herrors" @@ -172,8 +175,6 @@ func (commonResource) Slice(in interface{}) (interface{}, error) { return nil, fmt.Errorf("type %T is not a Resource", v) } groups[i] = g - { - } } return groups, nil default: @@ -197,8 +198,24 @@ type fileInfo interface { setSourceFilename(string) setSourceFs(afero.Fs) getFileInfo() hugofs.FileMetaInfo - hash() (string, error) size() int + hashProvider +} + +type hashProvider interface { + hash() string +} + +type resourceOrigin struct { + stale bool +} + +func (s *resourceOrigin) MarkStale() { + s.stale = true +} + +func (s *resourceOrigin) IsStale() bool { + return s.stale } // genericResource represents a generic linkable resource. @@ -235,7 +252,35 @@ func (l *genericResource) Data() interface{} { } func (l *genericResource) Key() string { - return l.RelPermalink() + // TODO1 consider repeating the section in the path segment. + + if l.fi != nil { + // Create a key that at least shares the base folder with the source, + // to facilitate effective cache busting on changes. + meta := l.fi.Meta() + p := meta.Path() + if p != "" { + d, _ := filepath.Split(p) + p = path.Join(d, l.relTargetDirFile.file) + key := memcache.CleanKey(p) + key = memcache.InsertKeyPathElements(key, meta.Component(), meta.Lang()) + + return key + } + } + + return memcache.CleanKey(l.RelPermalink()) + +} + +func (l *genericResource) IsStale() bool { + panic("TODO1") + +} + +func (l *genericResource) IsNotDependent(other identity.Identity) bool { + // TODO1 + return false } func (l *genericResource) MediaType() media.Type { @@ -417,6 +462,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour } fpath, fname := path.Split(u.targetPath) + r.resourcePathDescriptor.relTargetDirFile = dirFile{dir: fpath, file: fname} r.mergeData(u.data) @@ -623,26 +669,25 @@ func (fi *resourceFileInfo) setSourceFs(fs afero.Fs) { fi.sourceFs = fs } -func (fi *resourceFileInfo) hash() (string, error) { +func (fi *resourceFileInfo) hash() string { var err error fi.h.init.Do(func() { var hash string var f hugio.ReadSeekCloser f, err = fi.ReadSeekCloser() if err != nil { - err = errors.Wrap(err, "failed to open source file") - return + panic(fmt.Sprintf("failed to open source file: %s", err)) } defer f.Close() hash, err = helpers.MD5FromFileFast(f) if err != nil { - return + panic(err) } fi.h.value = hash }) - return fi.h.value, err + return fi.h.value } func (fi *resourceFileInfo) size() int { diff --git a/resources/resource/resources.go b/resources/resource/resources.go index ac5dd0b2b03..aecea48d616 100644 --- a/resources/resource/resources.go +++ b/resources/resource/resources.go @@ -20,10 +20,24 @@ import ( "github.com/gohugoio/hugo/hugofs/glob" ) +var ( + _ StaleInfo = Resources{} +) + // Resources represents a slice of resources, which can be a mix of different types. // I.e. both pages and images etc. type Resources []Resource +// Resources is stale if any of the the elements are stale. +func (rs Resources) IsStale() bool { + for _, r := range rs { + if s, ok := r.(StaleInfo); ok && s.IsStale() { + return true + } + } + return false +} + // ResourcesConverter converts a given slice of Resource objects to Resources. type ResourcesConverter interface { ToResources() Resources diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index f42372fa396..9a7564db8f3 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -15,6 +15,7 @@ package resource import ( "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources/images/exif" @@ -43,6 +44,10 @@ type Resource interface { ResourceMetaProvider ResourceParamsProvider ResourceDataProvider + + // Internal. Used in dependency tracking. + identity.IsNotDependentProvider + StaleInfo } // Image represents an image resource. @@ -179,6 +184,36 @@ type UnmarshableResource interface { Identifier } +// Staler controls stale state of a Resource. A stale resource should be discarded. +type Staler interface { + MarkStale() + StaleInfo +} + +// StaleInfo tells if a resource is marked as stale. +type StaleInfo interface { + IsStale() bool +} + +// IsStaleAny reports whether any of the os is marked as stale. +func IsStaleAny(os ...interface{}) bool { + for _, o := range os { + if s, ok := o.(StaleInfo); ok && s.IsStale() { + return true + } + } + return false +} + +// MarkStale will mark any of the oses as stale, if possible. +func MarkStale(os ...interface{}) { + for _, o := range os { + if s, ok := o.(Staler); ok { + s.MarkStale() + } + } +} + type resourceTypesHolder struct { mediaType media.Type resourceType string diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 6c4ba951bdf..f8f072159d8 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -16,26 +16,13 @@ package resources import ( "encoding/json" "io" - "path" - "path/filepath" - "regexp" - "strings" "sync" - "github.com/gohugoio/hugo/helpers" - - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/cache/filecache" - - "github.com/BurntSushi/locker" -) - -const ( - CACHE_CLEAR_ALL = "clear_all" - CACHE_OTHER = "other" ) type ResourceCache struct { @@ -43,123 +30,30 @@ type ResourceCache struct { sync.RWMutex - // Either resource.Resource or resource.Resources. - cache map[string]interface{} + // Memory cache with either + // resource.Resource or resource.Resources. + cache memcache.Getter fileCache *filecache.Cache - - // Provides named resource locks. - nlocker *locker.Locker -} - -// ResourceCacheKey converts the filename into the format used in the resource -// cache. -func ResourceCacheKey(filename string) string { - filename = filepath.ToSlash(filename) - return path.Join(resourceKeyPartition(filename), filename) -} - -func resourceKeyPartition(filename string) string { - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") - if ext == "" { - ext = CACHE_OTHER - } - return ext -} - -// Commonly used aliases and directory names used for some types. -var extAliasKeywords = map[string][]string{ - "sass": []string{"scss"}, - "scss": []string{"sass"}, -} - -// ResourceKeyPartitions resolves a ordered slice of partitions that is -// used to do resource cache invalidations. -// -// We use the first directory path element and the extension, so: -// a/b.json => "a", "json" -// b.json => "json" -// -// For some of the extensions we will also map to closely related types, -// e.g. "scss" will also return "sass". -// -func ResourceKeyPartitions(filename string) []string { - var partitions []string - filename = glob.NormalizePath(filename) - dir, name := path.Split(filename) - ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(name)), ".") - - if dir != "" { - partitions = append(partitions, strings.Split(dir, "/")[0]) - } - - if ext != "" { - partitions = append(partitions, ext) - } - - if aliases, found := extAliasKeywords[ext]; found { - partitions = append(partitions, aliases...) - } - - if len(partitions) == 0 { - partitions = []string{CACHE_OTHER} - } - - return helpers.UniqueStringsSorted(partitions) -} - -// ResourceKeyContainsAny returns whether the key is a member of any of the -// given partitions. -// -// This is used for resource cache invalidation. -func ResourceKeyContainsAny(key string, partitions []string) bool { - parts := strings.Split(key, "/") - for _, p1 := range partitions { - for _, p2 := range parts { - if p1 == p2 { - return true - } - } - } - return false } -func newResourceCache(rs *Spec) *ResourceCache { +func newResourceCache(rs *Spec, memCache *memcache.Cache) *ResourceCache { return &ResourceCache{ rs: rs, fileCache: rs.FileCaches.AssetsCache(), - cache: make(map[string]interface{}), - nlocker: locker.NewLocker(), + cache: memCache.GetOrCreatePartition("resources", memcache.ClearOnChange), } } -func (c *ResourceCache) clear() { - c.Lock() - defer c.Unlock() - - c.cache = make(map[string]interface{}) - c.nlocker = locker.NewLocker() -} - -func (c *ResourceCache) Contains(key string) bool { - key = c.cleanKey(filepath.ToSlash(key)) - _, found := c.get(key) - return found -} - -func (c *ResourceCache) cleanKey(key string) string { - return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") -} - -func (c *ResourceCache) get(key string) (interface{}, bool) { - c.RLock() - defer c.RUnlock() - r, found := c.cache[key] - return r, found -} - -func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, error)) (resource.Resource, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) +func (c *ResourceCache) GetOrCreate(key string, clearWhen memcache.ClearWhen, f func() (resource.Resource, error)) (resource.Resource, error) { + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + ClearWhen: clearWhen, + } + }) if r == nil || err != nil { return nil, err } @@ -167,43 +61,20 @@ func (c *ResourceCache) GetOrCreate(key string, f func() (resource.Resource, err } func (c *ResourceCache) GetOrCreateResources(key string, f func() (resource.Resources, error)) (resource.Resources, error) { - r, err := c.getOrCreate(key, func() (interface{}, error) { return f() }) + r, err := c.cache.GetOrCreate(key, func() memcache.Entry { + r, err := f() + return memcache.Entry{ + Value: r, + Err: err, + ClearWhen: memcache.ClearOnChange, + } + }) if r == nil || err != nil { return nil, err } return r.(resource.Resources), nil } -func (c *ResourceCache) getOrCreate(key string, f func() (interface{}, error)) (interface{}, error) { - key = c.cleanKey(key) - // First check in-memory cache. - r, found := c.get(key) - if found { - return r, nil - } - // This is a potentially long running operation, so get a named lock. - c.nlocker.Lock(key) - - // Double check in-memory cache. - r, found = c.get(key) - if found { - c.nlocker.Unlock(key) - return r, nil - } - - defer c.nlocker.Unlock(key) - - r, err := f() - if err != nil { - return nil, err - } - - c.set(key, r) - - return r, nil - -} - func (c *ResourceCache) getFilenames(key string) (string, string) { filenameMeta := key + ".json" filenameContent := key + ".content" @@ -256,56 +127,3 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) return fi, fc, err } - -func (c *ResourceCache) set(key string, r interface{}) { - c.Lock() - defer c.Unlock() - c.cache[key] = r -} - -func (c *ResourceCache) DeletePartitions(partitions ...string) { - partitionsSet := map[string]bool{ - // Always clear out the resources not matching any partition. - "other": true, - } - for _, p := range partitions { - partitionsSet[p] = true - } - - if partitionsSet[CACHE_CLEAR_ALL] { - c.clear() - return - } - - c.Lock() - defer c.Unlock() - - for k := range c.cache { - clear := false - for p := range partitionsSet { - if strings.Contains(k, p) { - // There will be some false positive, but that's fine. - clear = true - break - } - } - - if clear { - delete(c.cache, k) - } - } - -} - -func (c *ResourceCache) DeleteMatches(re *regexp.Regexp) { - c.Lock() - defer c.Unlock() - - for k := range c.cache { - if re.MatchString(k) { - delete(c.cache, k) - } - - } - -} diff --git a/resources/resource_cache_test.go b/resources/resource_cache_test.go deleted file mode 100644 index bcb24102594..00000000000 --- a/resources/resource_cache_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resources - -import ( - "path/filepath" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestResourceKeyPartitions(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - input string - expected []string - }{ - {"a.js", []string{"js"}}, - {"a.scss", []string{"sass", "scss"}}, - {"a.sass", []string{"sass", "scss"}}, - {"d/a.js", []string{"d", "js"}}, - {"js/a.js", []string{"js"}}, - {"D/a.JS", []string{"d", "js"}}, - {"d/a", []string{"d"}}, - {filepath.FromSlash("/d/a.js"), []string{"d", "js"}}, - {filepath.FromSlash("/d/e/a.js"), []string{"d", "js"}}, - } { - c.Assert(ResourceKeyPartitions(test.input), qt.DeepEquals, test.expected, qt.Commentf(test.input)) - } -} - -func TestResourceKeyContainsAny(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - key string - filename string - expected bool - }{ - {"styles/css", "asdf.css", true}, - {"styles/css", "styles/asdf.scss", true}, - {"js/foo.bar", "asdf.css", false}, - } { - c.Assert(ResourceKeyContainsAny(test.key, ResourceKeyPartitions(test.filename)), qt.Equals, test.expected) - } -} diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index 1ea92bea397..818dbaf5294 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -17,9 +17,10 @@ package bundler import ( "fmt" "io" - "path" "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" @@ -81,8 +82,7 @@ func (r *multiReadSeekCloser) Close() error { // Concat concatenates the list of Resource objects. func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resource, error) { - // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(targetPath, memcache.ClearOnRebuild, func() (resource.Resource, error) { var resolvedm media.Type // The given set of resources must be of the same Media Type. diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 4ac20d36e5a..49604ca0720 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -18,7 +18,8 @@ package create import ( "path" "path/filepath" - "strings" + + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/hugofs/glob" @@ -43,7 +44,7 @@ func New(rs *resources.Spec) *Client { // Get creates a new Resource by opening the given filename in the assets filesystem. func (c *Client) Get(filename string) (resource.Resource, error) { filename = filepath.Clean(filename) - return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(filename), func() (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(filename), memcache.ClearOnChange, func() (resource.Resource, error) { return c.rs.New(resources.ResourceSourceDescriptor{ Fs: c.rs.BaseFs.Assets.Fs, LazyPublish: true, @@ -74,13 +75,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro name = "__match" } - pattern = glob.NormalizePath(pattern) - partitions := glob.FilterGlobParts(strings.Split(pattern, "/")) - if len(partitions) == 0 { - partitions = []string{resources.CACHE_OTHER} - } - key := path.Join(name, path.Join(partitions...)) - key = path.Join(key, pattern) + key := path.Join(name, glob.NormalizePath(pattern)) return c.rs.ResourceCache.GetOrCreateResources(key, func() (resource.Resources, error) { var res resource.Resources @@ -116,7 +111,7 @@ func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, erro // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { - return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, targetPath), func() (resource.Resource, error) { + r, err := c.rs.ResourceCache.GetOrCreate(memcache.CleanKey(targetPath), memcache.ClearOnRebuild, func() (resource.Resource, error) { return c.rs.New( resources.ResourceSourceDescriptor{ Fs: c.rs.FileCaches.AssetsCache().Fs, @@ -128,4 +123,9 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro }) + if err == nil { + r.(resource.Staler).MarkStale() + } + return r, err + } diff --git a/resources/resource_metadata_test.go b/resources/resource_metadata_test.go index c79a5002121..0414127b92a 100644 --- a/resources/resource_metadata_test.go +++ b/resources/resource_metadata_test.go @@ -209,12 +209,12 @@ func TestAssignMetadata(t *testing.T) { }}, } { - foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) - logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) - foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) - logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) - foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) - logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) + foo2 = newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) + logo2 = newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) + foo1 = newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) + logo1 = newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) + foo3 = newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) + logo3 = newGenericResource(spec, nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) resources = resource.Resources{ foo2, diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 0ca60fe3136..bbbed9fa71f 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -23,6 +23,7 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/resources/jsconfig" "github.com/gohugoio/hugo/common/herrors" @@ -48,6 +49,7 @@ import ( func NewSpec( s *helpers.PathSpec, fileCaches filecache.Caches, + memCache *memcache.Cache, incr identity.Incrementer, logger loggers.Logger, errorHandler herrors.ErrorSender, @@ -94,11 +96,11 @@ func NewSpec( }, imageCache: newImageCache( fileCaches.ImageCache(), - + memCache, s, )} - rs.ResourceCache = newResourceCache(rs) + rs.ResourceCache = newResourceCache(rs, memCache) return rs, nil @@ -141,57 +143,11 @@ func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { return r.newResourceFor(fd) } -func (r *Spec) CacheStats() string { - r.imageCache.mu.RLock() - defer r.imageCache.mu.RUnlock() - - s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) - - count := 0 - for k := range r.imageCache.store { - if count > 5 { - break - } - s += "\n" + k - count++ - } - - return s -} - -func (r *Spec) ClearCaches() { - r.imageCache.clear() - r.ResourceCache.clear() -} - -func (r *Spec) DeleteBySubstring(s string) { - r.imageCache.deleteIfContains(s) -} - func (s *Spec) String() string { return "spec" } // TODO(bep) clean up below -func (r *Spec) newGenericResource(sourceFs afero.Fs, - targetPathBuilder func() page.TargetPaths, - osFileInfo os.FileInfo, - sourceFilename, - baseFilename string, - mediaType media.Type) *genericResource { - return r.newGenericResourceWithBase( - sourceFs, - nil, - nil, - targetPathBuilder, - osFileInfo, - sourceFilename, - baseFilename, - mediaType, - ) - -} - func (r *Spec) newGenericResourceWithBase( sourceFs afero.Fs, openReadSeekerCloser resource.OpenReadSeekCloser, diff --git a/resources/resource_test.go b/resources/resource_test.go index 7a0b8069d79..75ac94f79ce 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -34,7 +34,7 @@ func TestGenericResource(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) - r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo.css") @@ -48,11 +48,11 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := newTargetPaths("/foo") - r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) + r := newGenericResource(spec, nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) c.Assert(r.Permalink(), qt.Equals, "https://example.com/foo/foo.css") c.Assert(r.RelPermalink(), qt.Equals, "/foo/foo.css") - c.Assert(r.Key(), qt.Equals, "/foo/foo.css") + c.Assert(r.Key(), qt.Equals, "foo/foo.css") c.Assert(r.ResourceType(), qt.Equals, "css") } @@ -105,10 +105,10 @@ func TestResourcesByType(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), - spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo.png", "logo.css", pngType), + newGenericResource(spec, nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} c.Assert(len(resources.ByType("css")), qt.Equals, 3) c.Assert(len(resources.ByType("image")), qt.Equals, 1) @@ -119,11 +119,11 @@ func TestResourcesGetByPrefix(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} c.Assert(resources.GetMatch("asdf*"), qt.IsNil) c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -148,14 +148,14 @@ func TestResourcesGetMatch(t *testing.T) { c := qt.New(t) spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ - spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), - spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), - spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + newGenericResource(spec, nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), + newGenericResource(spec, nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), } c.Assert(resources.GetMatch("logo*").RelPermalink(), qt.Equals, "/logo1.png") @@ -212,7 +212,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := resource.Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)} + resources := resource.Resources{newGenericResource(spec, nil, nil, nil, "/a/"+a100, a100, media.CSSType)} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -228,17 +228,17 @@ func benchResources(b *testing.B) resource.Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) } return resources @@ -266,7 +266,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) + resources = append(resources, newGenericResource(spec, nil, nil, nil, "/a/"+name, name, media.CSSType)) } b.StartTimer() diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 8eacf7da4e6..3d7af38c23d 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -16,6 +16,8 @@ package htesting import ( "path/filepath" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -51,7 +53,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 0462f7ecdfc..3f5d6905f06 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -4,6 +4,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/cache/memcache" + "image" "io" "io/ioutil" @@ -90,7 +92,9 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + mc := memcache.New(memcache.Config{}) + + spec, err := NewSpec(s, filecaches, mc, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -129,7 +133,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, memcache.New(memcache.Config{}), nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir @@ -207,3 +211,22 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { t.Fatalf("Failed to write file: %s", err) } } + +func newGenericResource(r *Spec, sourceFs afero.Fs, + targetPathBuilder func() page.TargetPaths, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + nil, + nil, + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} diff --git a/resources/transform.go b/resources/transform.go index 354a20eece7..26bec6039ef 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -18,9 +18,13 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "sync" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/identity" + "github.com/pkg/errors" "github.com/gohugoio/hugo/resources/images/exif" @@ -42,6 +46,8 @@ var ( _ resource.ContentResource = (*resourceAdapter)(nil) _ resource.ReadSeekCloserResource = (*resourceAdapter)(nil) _ resource.Resource = (*resourceAdapter)(nil) + _ resource.Staler = (*resourceAdapterInner)(nil) + _ identity.IsNotDependentProvider = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) _ resource.ResourceMetaProvider = (*resourceAdapter)(nil) @@ -60,12 +66,17 @@ func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResour if lazyPublish { po = &publishOnce{} } + + s := &resourceOrigin{} + return &resourceAdapter{ resourceTransformations: &resourceTransformations{}, + resourceOrigin: s, resourceAdapterInner: &resourceAdapterInner{ - spec: spec, - publishOnce: po, - target: target, + spec: spec, + resourceOrigin: s, + publishOnce: po, + target: target, }, } } @@ -150,6 +161,11 @@ type publishOnce struct { type resourceAdapter struct { commonResource + + // This state is carried over into any clone of this adapter + // (when passed through a Hugo pipe). + *resourceOrigin + *resourceTransformations *resourceAdapterInner } @@ -188,8 +204,11 @@ func (r *resourceAdapter) Exif() *exif.Exif { } func (r *resourceAdapter) Key() string { - r.init(false, false) - return r.target.(resource.Identifier).Key() + return r.TransformationKey() +} + +func (r *resourceAdapter) IsNotDependent(other identity.Identity) bool { + panic("TODO1: resourceAdapter") } func (r *resourceAdapter) MediaType() media.Type { @@ -252,9 +271,10 @@ func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransfo } r.resourceAdapterInner = &resourceAdapterInner{ - spec: r.spec, - publishOnce: &publishOnce{}, - target: r.target, + spec: r.spec, + resourceOrigin: r.resourceOrigin, + publishOnce: &publishOnce{}, + target: r.target, } return &r, nil @@ -297,6 +317,57 @@ func (r *resourceAdapter) publish() { } func (r *resourceAdapter) TransformationKey() string { + r.transformationsKeyInit.Do(func() { + if len(r.transformations) == 0 { + r.transformationsKey = r.target.Key() + return + } + + var adder string + for _, tr := range r.transformations { + adder = adder + "_" + tr.Key().Value() + } + + key := r.target.Key() + adder = "_" + helpers.MD5String(adder) + + // Preserve any file extension if possible. + dotIdx := strings.LastIndex(key, ".") + if dotIdx == -1 { + key += adder + } else { + key = key[:dotIdx] + adder + key[dotIdx:] + } + + key = memcache.CleanKey(key) + r.transformationsKey = key + }) + + return r.transformationsKey +} + +// We changed the format of the resource cache keys in Hugo v0.77. +// To reduce the nois, especially on the theme site, we fall back to reading +// files on the old format. +// TODO(bep) eventually remove. +func (r *resourceAdapter) transformationKeyV074() string { + cleanKey := func(key string) string { + return strings.TrimPrefix(path.Clean(strings.ToLower(key)), "/") + } + + resourceKeyPartition := func(filename string) string { + ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") + if ext == "" { + ext = "other" + } + return ext + } + + resourceCacheKey := func(filename string) string { + filename = filepath.ToSlash(filename) + return path.Join(resourceKeyPartition(filename), filename) + } + // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. var key string @@ -304,35 +375,34 @@ func (r *resourceAdapter) TransformationKey() string { key = key + "_" + tr.Key().Value() } - base := ResourceCacheKey(r.target.Key()) - return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key) -} + base := resourceCacheKey(r.target.RelPermalink()) + return cleanKey(base) + "_" + helpers.MD5String(key) -func (r *resourceAdapter) transform(publish, setContent bool) error { - cache := r.spec.ResourceCache +} +func (r *resourceAdapter) getOrTransform(publish, setContent bool) error { key := r.TransformationKey() + res, err := r.spec.ResourceCache.cache.GetOrCreate(key, func() memcache.Entry { + r, err := r.transform(key, publish, setContent) + return memcache.Entry{ + Value: r, + Err: err, + ClearWhen: memcache.ClearOnChange, + } + }) - cached, found := cache.get(key) - - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - return nil + if err != nil { + return err } - // Acquire a write lock for the named transformation. - cache.nlocker.Lock(key) - // Check the cache again. - cached, found = cache.get(key) - if found { - r.resourceAdapterInner = cached.(*resourceAdapterInner) - cache.nlocker.Unlock(key) - return nil - } + r.resourceAdapterInner = res.(*resourceAdapterInner) - defer cache.nlocker.Unlock(key) - defer cache.set(key, r.resourceAdapterInner) + return nil + +} +func (r *resourceAdapter) transform(key string, publish, setContent bool) (*resourceAdapterInner, error) { + cache := r.spec.ResourceCache b1 := bp.GetBuffer() b2 := bp.GetBuffer() defer bp.PutBuffer(b1) @@ -353,7 +423,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { contentrc, err := contentReadSeekerCloser(r.target) if err != nil { - return err + return nil, err } defer contentrc.Close() @@ -426,21 +496,25 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { } else { err = tr.Transform(tctx) if err != nil && err != herrors.ErrFeatureNotAvailable { - return newErr(err) + return nil, newErr(err) } if mayBeCachedOnDisk { tryFileCache = r.spec.BuildConfig.UseResourceCache(err) } if err != nil && !tryFileCache { - return newErr(err) + return nil, newErr(err) } } if tryFileCache { f := r.target.tryTransformedFileCache(key, updates) if f == nil { - return newErr(errors.Errorf("resource %q not found in file cache", key)) + keyOldFormat := r.transformationKeyV074() + f = r.target.tryTransformedFileCache(keyOldFormat, updates) + if f == nil { + return nil, newErr(errors.Errorf("resource %q not found in file cache", key)) + } } transformedContentr = f updates.sourceFs = cache.fileCache.Fs @@ -465,7 +539,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { if publish { publicw, err := r.target.openPublishFileForWriting(updates.targetPath) if err != nil { - return err + return nil, err } publishwriters = append(publishwriters, publicw) } @@ -475,7 +549,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { // Also write it to the cache fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata()) if err != nil { - return err + return nil, err } updates.sourceFilename = &fi.Name updates.sourceFs = cache.fileCache.Fs @@ -506,7 +580,7 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { publishw := hugio.NewMultiWriteCloser(publishwriters...) _, err = io.Copy(publishw, transformedContentr) if err != nil { - return err + return nil, err } publishw.Close() @@ -517,11 +591,11 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { newTarget, err := r.target.cloneWithUpdates(updates) if err != nil { - return err + return nil, err } r.target = newTarget - return nil + return r.resourceAdapterInner, nil } func (r *resourceAdapter) init(publish, setContent bool) { @@ -541,7 +615,7 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { r.publishOnce = nil } - r.transformationsErr = r.transform(publish, setContent) + r.transformationsErr = r.getOrTransform(publish, setContent) if r.transformationsErr != nil { if r.spec.ErrorSender != nil { r.spec.ErrorSender.SendError(r.transformationsErr) @@ -559,16 +633,28 @@ func (r *resourceAdapter) initTransform(publish, setContent bool) { type resourceAdapterInner struct { target transformableResource + *resourceOrigin + spec *Spec // Handles publishing (to /public) if needed. *publishOnce } +func (r *resourceAdapterInner) ResourceTarget() resource.Resource { + return r.target +} + +func (r *resourceAdapterInner) IsNotDependent(other identity.Identity) bool { + return r.target.IsNotDependent(other) +} + type resourceTransformations struct { - transformationsInit sync.Once - transformationsErr error - transformations []ResourceTransformation + transformationsInit sync.Once + transformationsErr error + transformationsKeyInit sync.Once + transformationsKey string + transformations []ResourceTransformation } type transformableResource interface { @@ -603,7 +689,6 @@ func (u *transformationUpdate) toTransformedResourceMetadata() transformedResour } func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) { - u.targetPath = ctx.OutPath u.mediaType = ctx.OutMediaType u.data = ctx.Data u.targetPath = ctx.InPath diff --git a/tpl/openapi/openapi3/openapi3.go b/tpl/openapi/openapi3/openapi3.go index 7dfd2f6a797..b2671ad2a1e 100644 --- a/tpl/openapi/openapi3/openapi3.go +++ b/tpl/openapi/openapi3/openapi3.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" kopenapi3 "github.com/getkin/kin-openapi/openapi3" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/resources/resource" @@ -29,22 +29,15 @@ import ( // New returns a new instance of the openapi3-namespaced template functions. func New(deps *deps.Deps) *Namespace { - // TODO1 consolidate when merging that "other branch" -- but be aware of the keys. - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - return &Namespace{ - cache: cache, + cache: deps.MemCache.GetOrCreatePartition("tpl/openapi3", memcache.ClearOnChange), deps: deps, } } // Namespace provides template functions for the "openapi3". type Namespace struct { - cache *namedmemcache.Cache + cache memcache.Getter deps *deps.Deps } @@ -55,21 +48,22 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg return nil, errors.New("no Key set in Resource") } - v, err := ns.cache.GetOrCreate(key, func() (interface{}, error) { + v, err := ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } + defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } s := &kopenapi3.Swagger{} @@ -80,12 +74,19 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagg err = metadecoders.Default.UnmarshalTo(b, f, s) } if err != nil { - return nil, err + return memcache.Entry{Err: err} } err = kopenapi3.NewSwaggerLoader().ResolveRefsIn(s, nil) - return s, err + return memcache.Entry{ + Value: s, + Err: err, + ClearWhen: memcache.ClearOnChange, + StaleFunc: func() bool { + return resource.IsStaleAny(r) + }, + } }) if err != nil { diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index e3a4ce09029..0705cfe0b03 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -111,7 +111,7 @@ func needsBaseTemplate(templ string) bool { } func newIdentity(name string) identity.Manager { - return identity.NewManager(identity.NewPathIdentity(files.ComponentFolderLayouts, name)) + return identity.NewManager(identity.NewPathIdentity(files.ComponentFolderLayouts, name, "", "")) } func newStandaloneTextTemplate(funcs map[string]interface{}) tpl.TemplateParseFinder { @@ -415,7 +415,7 @@ func (t *templateHandler) findLayout(d output.LayoutDescriptor, f output.Format) ts.baseInfo = base // Add the base identity to detect changes - ts.Add(identity.NewPathIdentity(files.ComponentFolderLayouts, base.name)) + ts.Add(identity.NewPathIdentity(files.ComponentFolderLayouts, base.name, "", "")) } t.applyTemplateTransformers(t.main, ts) diff --git a/tpl/transform/init_test.go b/tpl/transform/init_test.go index 47bd8a391e6..393b6b140d8 100644 --- a/tpl/transform/init_test.go +++ b/tpl/transform/init_test.go @@ -16,6 +16,8 @@ package transform import ( "testing" + "github.com/gohugoio/hugo/cache/memcache" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/htesting/hqt" @@ -28,7 +30,9 @@ func TestInit(t *testing.T) { var ns *internal.TemplateFuncsNamespace for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns = nsf(&deps.Deps{}) + ns = nsf(&deps.Deps{ + MemCache: memcache.New(memcache.Config{}), + }) if ns.Name == name { found = true break diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index b168d2a50d4..e4ea17beba2 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -18,7 +18,7 @@ import ( "html" "html/template" - "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/cache/memcache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" @@ -27,22 +27,19 @@ import ( // New returns a new instance of the transform-namespaced template functions. func New(deps *deps.Deps) *Namespace { - cache := namedmemcache.New() - deps.BuildStartListeners.Add( - func() { - cache.Clear() - }) - + if deps.MemCache == nil { + panic("must provide MemCache") + } return &Namespace{ - cache: cache, deps: deps, + cache: deps.MemCache.GetOrCreatePartition("tpl/transform", memcache.ClearOnChange), } } // Namespace provides template functions for the "transform" namespace. type Namespace struct { - cache *namedmemcache.Cache deps *deps.Deps + cache memcache.Getter } // Emojify returns a copy of s with all emoji codes replaced with actual emojis. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index b3f4206ff6b..1a14661f246 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -16,6 +16,8 @@ package transform import ( "html/template" + "github.com/gohugoio/hugo/cache/memcache" + "testing" "github.com/gohugoio/hugo/common/loggers" @@ -251,6 +253,7 @@ func newDeps(cfg config.Provider) *deps.Deps { return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), + MemCache: memcache.New(memcache.Config{}), ContentSpec: cs, } } diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index b606c870aa7..3288b9b03b2 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -17,6 +17,8 @@ import ( "io/ioutil" "strings" + "github.com/gohugoio/hugo/cache/memcache" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/common/types" @@ -69,24 +71,33 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key += decoder.OptionsKey() } - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := metadecoders.FormatFromMediaType(r.MediaType()) if f == "" { - return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + return memcache.Entry{Err: errors.Errorf("MIME %q not supported", r.MediaType())} } reader, err := r.ReadSeekCloser() if err != nil { - return nil, err + return memcache.Entry{Err: err} } defer reader.Close() b, err := ioutil.ReadAll(reader) if err != nil { - return nil, err + return memcache.Entry{Err: err} } - return decoder.Unmarshal(b, f) + v, err := decoder.Unmarshal(b, f) + + return memcache.Entry{ + Value: v, + Err: err, + ClearWhen: memcache.ClearOnChange, + StaleFunc: func() bool { + return resource.IsStaleAny(r) + }, + } }) } @@ -97,13 +108,15 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { key := helpers.MD5String(dataStr) - return ns.cache.GetOrCreate(key, func() (interface{}, error) { + return ns.cache.GetOrCreate(key, func() memcache.Entry { f := decoder.FormatFromContentString(dataStr) if f == "" { - return nil, errors.New("unknown format") + return memcache.Entry{Err: errors.New("unknown format")} } - return decoder.Unmarshal([]byte(dataStr), f) + v, err := decoder.Unmarshal([]byte(dataStr), f) + + return memcache.Entry{Value: v, Err: err, ClearWhen: memcache.ClearOnChange} }) } diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 183bdefd5bc..8796cef3724 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -145,7 +145,7 @@ a;b;c`, mime: media.CSVType}, map[string]interface{}{"DElimiter": ";", "Comment" {tstNoStringer{}, nil, false}, } { - ns.cache.Clear() + ns.deps.MemCache.Clear() var args []interface{}