Skip to content

Commit

Permalink
Make cache memory bound
Browse files Browse the repository at this point in the history
This commit also consolidates all (or the most important) memory caches in Hugo.

Fixes gohugoio#7425
  • Loading branch information
bep committed Jun 25, 2020
1 parent 057b137 commit 24925c7
Show file tree
Hide file tree
Showing 20 changed files with 293 additions and 334 deletions.
108 changes: 108 additions & 0 deletions cache/memcache/memcache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// Copyright 2020 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package memcache provides the core memory cache used in Hugo.
package memcache

import (
"time"

"github.com/BurntSushi/locker"
"github.com/karlseguin/ccache"
)

// Cache configures a cache.
type Cache struct {
cache *ccache.LayeredCache

ttl time.Duration
nlocker *locker.Locker
}

type cacheEntry struct {
size int64
value interface{}
err error
}

func (c cacheEntry) Size() int64 {
return c.size
}

// New creates a new cache.
func New() *Cache {
return &Cache{
// TODO1
cache: ccache.Layered(ccache.Configure().MaxSize(100).ItemsToPrune(100)),
ttl: time.Second * 5,
nlocker: locker.NewLocker(),
}
}

// Clear clears the cache state.
// This method is not thread safe.
func (c *Cache) Clear() {
c.nlocker = locker.NewLocker()
c.cache.Clear()
}

func (c *Cache) Has(primary, secondary string) bool {
return c.cache.Get(primary, secondary) != nil
}

func (c *Cache) Get(primary, secondary string) (interface{}, bool) {
v := c.cache.Get(primary, secondary)
if v == nil {
return nil, false
}
return v.Value(), true
}

func (c *Cache) DeleteAll(primary string) bool {
return c.cache.DeleteAll(primary)
}

func (c *Cache) Stop() {
c.cache.Stop()
}

// GetOrCreate tries to get the value with the given cache keys, if not found
// create will be called and cached.
// This method is thread safe.
func (c *Cache) GetOrCreate(primary, secondary string, create func() (interface{}, error)) (interface{}, error) {
if v := c.cache.Get(primary, secondary); v != nil {
entry := v.Value().(cacheEntry)
return entry.value, entry.err
}

// The provided create function may be a relatively time consuming operation,
// and there will in the commmon case be concurrent requests for the same key'd
// resource, so make sure we pause these until the result is ready.
key := primary + secondary
c.nlocker.Lock(key)
defer c.nlocker.Unlock(key)

// Try again.
if v := c.cache.Get(primary, secondary); v != nil {
entry := v.Value().(cacheEntry)
return entry.value, entry.err
}

// Create it and store it in cache.
value, err := create()

// TODO1 size
c.cache.Set(primary, secondary, cacheEntry{value: value, err: err, size: 1}, c.ttl)

return value, err
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
// Copyright 2020 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -11,7 +11,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package namedmemcache
package memcache

import (
"fmt"
Expand All @@ -21,7 +21,7 @@ import (
qt "github.com/frankban/quicktest"
)

func TestNamedCache(t *testing.T) {
func TesCache(t *testing.T) {
t.Parallel()
c := qt.New(t)

Expand All @@ -34,17 +34,17 @@ func TestNamedCache(t *testing.T) {
}

for i := 0; i < 5; i++ {
v1, err := cache.GetOrCreate("a1", create)
v1, err := cache.GetOrCreate("a", "a1", create)
c.Assert(err, qt.IsNil)
c.Assert(v1, qt.Equals, 1)
v2, err := cache.GetOrCreate("a2", create)
v2, err := cache.GetOrCreate("a", "a2", create)
c.Assert(err, qt.IsNil)
c.Assert(v2, qt.Equals, 2)
}

cache.Clear()

v3, err := cache.GetOrCreate("a2", create)
v3, err := cache.GetOrCreate("a", "a2", create)
c.Assert(err, qt.IsNil)
c.Assert(v3, qt.Equals, 3)
}
Expand All @@ -70,7 +70,7 @@ func TestNamedCacheConcurrent(t *testing.T) {
defer wg.Done()
for j := 0; j < 100; j++ {
id := fmt.Sprintf("id%d", j)
v, err := cache.GetOrCreate(id, create(j))
v, err := cache.GetOrCreate("a", id, create(j))
c.Assert(err, qt.IsNil)
c.Assert(v, qt.Equals, j)
}
Expand Down
79 changes: 0 additions & 79 deletions cache/namedmemcache/named_cache.go

This file was deleted.

20 changes: 17 additions & 3 deletions deps/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"sync/atomic"
"time"

"github.com/gohugoio/hugo/cache/memcache"

"github.com/pkg/errors"

"github.com/gohugoio/hugo/cache/filecache"
Expand Down Expand Up @@ -62,9 +64,12 @@ type Deps struct {
// The configuration to use
Cfg config.Provider `json:"-"`

// The file cache to use.
// The file caches to use.
FileCaches filecache.Caches

// The memory cache to use.
MemCache *memcache.Cache

// The translation func to use
Translate func(translationID string, args ...interface{}) string `json:"-"`

Expand Down Expand Up @@ -158,6 +163,13 @@ type ResourceProvider interface {
Clone(deps *Deps) error
}

// Stop stops all running caches etc.
func (d *Deps) Stop() {
if d.MemCache != nil {
d.MemCache.Stop()
}
}

func (d *Deps) Tmpl() tpl.TemplateHandler {
return d.tmpl
}
Expand Down Expand Up @@ -236,11 +248,12 @@ func New(cfg DepsCfg) (*Deps, error) {
if err != nil {
return nil, errors.WithMessage(err, "failed to create file caches from configuration")
}
memCache := memcache.New()

errorHandler := &globalErrHandler{}
buildState := &BuildState{}

resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes)
resourceSpec, err := resources.NewSpec(ps, fileCaches, memCache, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -277,6 +290,7 @@ func New(cfg DepsCfg) (*Deps, error) {
Language: cfg.Language,
Site: cfg.Site,
FileCaches: fileCaches,
MemCache: memCache,
BuildStartListeners: &Listeners{},
BuildState: buildState,
Timeout: time.Duration(timeoutms) * time.Millisecond,
Expand Down Expand Up @@ -311,7 +325,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er
// The resource cache is global so reuse.
// TODO(bep) clean up these inits.
resourceCache := d.ResourceSpec.ResourceCache
d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes)
d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.MemCache, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes)
if err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ require (
github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93
github.com/gorilla/websocket v1.4.1
github.com/jdkato/prose v1.1.1
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/kr/pretty v0.2.0 // indirect
github.com/kyokomi/emoji v2.2.1+incompatible
github.com/magefile/mage v1.9.0
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,9 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/karlseguin/ccache v1.0.1 h1:0gpC6z1qtv0cKmsi5Su5tTB6bJ2vm9bfOLACpDEB/Ro=
github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down
1 change: 1 addition & 0 deletions hugolib/hugo_sites.go
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,7 @@ func (h *HugoSites) reset(config *BuildCfg) {
}

h.init.Reset()
h.MemCache.Clear()
}

// resetLogs resets the log counters etc. Used to do a new build on the same sites.
Expand Down
4 changes: 4 additions & 0 deletions hugolib/hugo_sites_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
// Make sure we don't trigger rebuilds in parallel.
h.runningMu.Lock()
defer h.runningMu.Unlock()
} else {
defer func() {
h.Stop()
}()
}

ctx, task := trace.NewTask(context.Background(), "Build")
Expand Down
11 changes: 7 additions & 4 deletions hugolib/hugo_sites_build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -417,15 +417,18 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {

func TestMultiSitesRebuild(t *testing.T) {
// t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4
// This leaktest seems to be a little bit shaky on Travis.
if !isCI() {
defer leaktest.CheckTimeout(t, 10*time.Second)()
}

c := qt.New(t)

b := newMultiSiteTestDefaultBuilder(t).Running().CreateSites().Build(BuildCfg{})

defer b.H.Stop()

// This leaktest seems to be a little bit shaky on Travis.
if !isCI() {
defer leaktest.CheckTimeout(t, 10*time.Second)()
}

sites := b.H.Sites
fs := b.Fs

Expand Down
Loading

0 comments on commit 24925c7

Please sign in to comment.