Add a file cache

Fixes gohugoio#5404

cache/filecache/filecache.go

@@ -0,0 +1,227 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/BurntSushi/locker"
+	"github.com/bep/mapstructure"
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+)
+
+const cachesConfigKey = "caches"
+
+var defaultCacheConfig = cacheConfig{
+	TTL: -1,
+	Dir: ":cacheDir",
+}
+
+var defaultCacheConfigs = map[string]cacheConfig{
+	"getjson": defaultCacheConfig,
+	"getcsv":  defaultCacheConfig,
+}
+
+type cachesConfig map[string]cacheConfig
+
+type cacheConfig struct {
+	// Time to Live. Any items older than this will be removed and
+	// not returned from the cache.
+	// -1 means forever.
+	TTL int
+
+	// The directory where files are stored.
+	Dir string
+}
+
+// Cache caches a set of files in a directory. This is usually a file on
+// disk, but since this is backed by an Afero file system, it can be anything.
+type Cache struct {
+	fs afero.Fs
+
+	// Time to live, in seconds
+	ttl int
+
+	nlocker *locker.Locker
+}
+
+// NewCache creates a new file cache with the given filesystem and TTL.
+func NewCache(fs afero.Fs, ttl int) *Cache {
+	return &Cache{
+		fs:      fs,
+		nlocker: locker.NewLocker(),
+		ttl:     ttl,
+	}
+}
+
+// GetOrCreate tries to get the named file from cache. If not found or expired, f will
+// be invoked and the result cached.
+// This method is protected by a named lock using the given name as identifier.
+func (c *Cache) GetOrCreate(name string, f func() (io.ReadCloser, error)) (io.ReadCloser, error) {
+	name = filepath.Clean(name)
+	c.nlocker.RLock(name)
+	r, expired := c.get(name)
+	if !expired && r != nil {
+		c.nlocker.RUnlock(name)
+		return r, nil
+	}
+	c.nlocker.RUnlock(name)
+
+	// Need a write lock for the rest.
+	c.nlocker.Lock(name)
+	defer c.nlocker.Unlock(name)
+
+	// Double check.
+	r, expired = c.get(name)
+	if !expired && r != nil {
+		return r, nil
+	}
+
+	if expired {
+		c.fs.Remove(name)
+	}
+
+	nr, err := f()
+	if err != nil {
+		return nil, err
+	}
+
+	if c.ttl == 0 {
+		// No caching.
+		return struct {
+			io.Reader
+			io.Closer
+		}{
+			nr,
+			ioutil.NopCloser(nil),
+		}, nil
+	}
+
+	var buff bytes.Buffer
+
+	return struct {
+		io.Reader
+		io.Closer
+	}{
+		&buff,
+		ioutil.NopCloser(nil),
+	}, afero.WriteReader(c.fs, name, io.TeeReader(nr, &buff))
+
+}
+
+func (c *Cache) get(name string) (hugio.ReadSeekCloser, bool) {
+	if c.ttl == 0 {
+		return nil, false
+	}
+
+	if c.ttl > 0 {
+		fi, err := c.fs.Stat(name)
+		if err != nil {
+			return nil, false
+		}
+
+		expiry := time.Now().Add(-time.Duration(c.ttl) * time.Second)
+		expired := fi.ModTime().Before(expiry)
+		if expired {
+			return nil, true
+		}
+	}
+
+	f, err := c.fs.Open(name)
+	if err != nil {
+		return nil, false
+	}
+	return f, false
+}
+
+type Caches map[string]*Cache
+
+// Get gets a named cache, nil if none found.
+func (f Caches) Get(name string) *Cache {
+	return f[strings.ToLower(name)]
+}
+
+// NewCachesFromConfig creates a new set of file caches from the given
+// configuration.
+func NewCachesFromConfig(fs afero.Fs, cfg config.Provider) (Caches, error) {
+	dcfg, err := decodeConfig(fs, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(Caches)
+	for k, v := range dcfg {
+		// TODO(bep) cache placeholders + CI?
+		baseDir := filepath.Join(k, v.Dir)
+		bfs := afero.NewBasePathFs(fs, baseDir)
+		m[k] = NewCache(bfs, v.TTL)
+	}
+
+	return m, nil
+}
+
+func decodeConfig(fs afero.Fs, cfg config.Provider) (cachesConfig, error) {
+	c := make(cachesConfig)
+	// Add defaults
+	for k, v := range defaultCacheConfigs {
+		c[k] = v
+	}
+
+	if !cfg.IsSet(cachesConfigKey) {
+		return c, nil
+	}
+
+	m := cfg.GetStringMap(cachesConfigKey)
+
+	for k, v := range m {
+		cc := defaultCacheConfig
+
+		if err := mapstructure.WeakDecode(v, &cc); err != nil {
+			return nil, err
+		}
+
+		if cc.Dir == "" {
+			return c, errors.New("must provide cache Dir")
+		}
+
+		c[strings.ToLower(k)] = cc
+	}
+
+	cacheDir := cfg.GetString("cacheDir")
+	if cacheDir == "" {
+		var err error
+		cacheDir, err = afero.TempDir(fs, "hugo_cache", "")
+		if err != nil {
+			return c, err
+		}
+	}
+
+	// Expand dir variables
+	// TODO(bep) cache
+	for k, v := range c {
+		v.Dir = strings.Replace(v.Dir, ":cacheDir", cacheDir, 1)
+		c[k] = v
+	}
+
+	return c, nil
+}

cache/filecache/filecache_test.go

@@ -0,0 +1,169 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/spf13/afero"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileCache(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.concurrent]
+ttl = 111
+dir = "/cache/c"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	caches, err := NewCachesFromConfig(afero.NewMemMapFs(), cfg)
+	assert.NoError(err)
+
+	const cacheName = "Concurrent"
+
+	c := caches.Get(cacheName)
+	assert.NotNil(c)
+	assert.Equal(111, c.ttl)
+
+	r, err := c.GetOrCreate("a", func() (io.ReadCloser, error) {
+		return struct {
+			io.ReadSeeker
+			io.Closer
+		}{
+			strings.NewReader("abc"),
+			ioutil.NopCloser(nil),
+		}, nil
+	})
+
+	assert.NoError(err)
+	assert.NotNil(r)
+	b, _ := ioutil.ReadAll(r)
+	r.Close()
+	assert.Equal("abc", string(b))
+
+	assert.NotNil(caches.Get(strings.ToUpper(cacheName)))
+
+}
+
+func TestFileCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.concurrent]
+ttl = 1
+dir = "/cache/c"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	caches, err := NewCachesFromConfig(afero.NewMemMapFs(), cfg)
+	assert.NoError(err)
+
+	const cacheName = "concurrent"
+
+	filenameData := func(i int) (string, string) {
+		data := fmt.Sprintf("data: %d", i)
+		filename := fmt.Sprintf("file%d", i)
+		return filename, data
+	}
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 20; j++ {
+				c := caches.Get(cacheName)
+				assert.NotNil(c)
+				filename, data := filenameData(j)
+				r, err := c.GetOrCreate(filename, func() (io.ReadCloser, error) {
+					return struct {
+						io.ReadSeeker
+						io.Closer
+					}{
+						strings.NewReader(data),
+						ioutil.NopCloser(nil),
+					}, nil
+				})
+				assert.NoError(err)
+				b, _ := ioutil.ReadAll(r)
+				r.Close()
+				assert.Equal(data, string(b))
+				// Trigger som expiration.
+				time.Sleep(200 * time.Millisecond)
+			}
+		}()
+
+	}
+	wg.Wait()
+}
+
+func TestDecodeConfig(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.c1]
+ttl = 1234
+dir = "/path/to/c1"
+[caches.c2]
+ttl = 3456
+dir = "/path/to/c2"
+[caches.c3]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(afero.NewMemMapFs(), cfg)
+	assert.NoError(err)
+
+	assert.Equal(5, len(decoded))
+
+	c2 := decoded["c2"]
+	assert.Equal(3456, c2.TTL)
+	assert.Equal("/path/to/c2", c2.Dir)
+
+	c3 := decoded["c3"]
+	assert.Equal(-1, c3.TTL)
+	assert.Equal("/path/to/c3", c3.Dir)
+
+}