Add a file cache

Fixes gohugoio#5404

cache/filecache/filecache.go

@@ -0,0 +1,223 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"io"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/BurntSushi/locker"
+	"github.com/bep/mapstructure"
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+)
+
+const cachesConfigKey = "caches"
+
+var defaultCacheConfig = cacheConfig{
+	TTL: -1,
+	Dir: ":cacheDir",
+}
+
+var defaultCacheConfigs = map[string]cacheConfig{
+	"getjson": defaultCacheConfig,
+	"getcsv":  defaultCacheConfig,
+}
+
+type cachesConfig map[string]cacheConfig
+
+type cacheConfig struct {
+	// Time to Live. Any items older than this will be removed and
+	// not returned from the cache.
+	// -1 means forever.
+	TTL int
+
+	// The directory where files are stored.
+	Dir string
+}
+
+// Cache caches a set of files in a directory. This is usually a file on
+// disk, but since this is backed by an Afero file system, it can be anything.
+type Cache struct {
+	fs afero.Fs
+
+	// Time to live, in seconds
+	ttl int
+
+	nlocker *locker.Locker
+}
+
+// NewCache creates a new file cache with the given filesystem and TTL.
+func NewCache(fs afero.Fs, ttl int) *Cache {
+	return &Cache{
+		fs:      fs,
+		nlocker: locker.NewLocker(),
+		ttl:     ttl,
+	}
+}
+
+// Get gets a file from the cache given a filename. It will return nil
+// if file was not found in cache or if it's expired.
+// TODO(bep) cache ignoreCache
+func (c *Cache) Get(filename string) hugio.ReadSeekCloser {
+	if c.ttl == 0 {
+		return nil
+	}
+
+	filename = filepath.Clean(filename)
+
+	c.nlocker.RLock(filename)
+
+	fi, err := c.fs.Stat(filename)
+	if err != nil {
+		c.nlocker.RUnlock(filename)
+		return nil
+	}
+
+	if c.ttl > 0 {
+		expiry := time.Now().Add(-time.Duration(c.ttl) * time.Second)
+		expired := fi.ModTime().Before(expiry)
+
+		if expired {
+			// Need a write lock for this.
+			c.nlocker.RUnlock(filename)
+			c.nlocker.Lock(filename)
+
+			// Double check
+			fi, err := c.fs.Stat(filename)
+			expired := err == nil && fi.ModTime().Before(expiry)
+			if expired {
+				c.fs.Remove(filename)
+			}
+
+			c.nlocker.Unlock(filename)
+
+			if err != nil || expired {
+				return nil
+			}
+
+			c.nlocker.RLock(filename)
+		}
+	}
+
+	defer c.nlocker.RUnlock(filename)
+
+	f, err := c.fs.Open(filename)
+	if err != nil {
+		return nil
+	}
+	return f
+}
+
+// WriteReader writes r to filename in the file cache.
+func (c *Cache) WriteReader(filename string, r io.Reader) error {
+	filename = filepath.Clean(filename)
+	c.nlocker.Lock(filename)
+	defer c.nlocker.Unlock(filename)
+
+	return afero.WriteReader(c.fs, filename, r)
+}
+
+// WriteFunc writes the io.Reader returned from f. This can be used for
+// long running operations, as it will prevent others from reading the
+// cached file until it's ready.
+func (c *Cache) WriteFunc(filename string, f func() (io.Reader, error)) error {
+	filename = filepath.Clean(filename)
+	c.nlocker.Lock(filename)
+	defer c.nlocker.Unlock(filename)
+
+	r, err := f()
+	if err != nil {
+		return err
+	}
+
+	return afero.WriteReader(c.fs, filename, r)
+}
+
+type Caches map[string]*Cache
+
+// Get gets a named cache, nil if none found.
+func (f Caches) Get(name string) *Cache {
+	return f[strings.ToLower(name)]
+}
+
+// NewCachesFromConfig creates a new set of file caches from the given
+// configuration.
+func NewCachesFromConfig(fs afero.Fs, cfg config.Provider) (Caches, error) {
+	dcfg, err := decodeConfig(fs, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(Caches)
+	for k, v := range dcfg {
+		// TODO(bep) cache placeholders + CI?
+		baseDir := filepath.Join(k, v.Dir)
+		bfs := afero.NewBasePathFs(fs, baseDir)
+		m[k] = NewCache(bfs, v.TTL)
+	}
+
+	return m, nil
+}
+
+func decodeConfig(fs afero.Fs, cfg config.Provider) (cachesConfig, error) {
+	c := make(cachesConfig)
+	// Add defaults
+	for k, v := range defaultCacheConfigs {
+		c[k] = v
+	}
+
+	if !cfg.IsSet(cachesConfigKey) {
+		return c, nil
+	}
+
+	m := cfg.GetStringMap(cachesConfigKey)
+
+	for k, v := range m {
+		cc := defaultCacheConfig
+
+		if err := mapstructure.WeakDecode(v, &cc); err != nil {
+			return nil, err
+		}
+
+		if cc.Dir == "" {
+			return c, errors.New("must provide cache Dir")
+		}
+
+		c[strings.ToLower(k)] = cc
+	}
+
+	cacheDir := cfg.GetString("cacheDir")
+	if cacheDir == "" {
+		var err error
+		cacheDir, err = afero.TempDir(fs, "hugo_cache", "")
+		if err != nil {
+			return c, err
+		}
+	}
+
+	// Expand dir variables
+	// TODO(bep) cache
+	for k, v := range c {
+		v.Dir = strings.Replace(v.Dir, ":cacheDir", cacheDir, 1)
+		c[k] = v
+	}
+
+	return c, nil
+}

cache/filecache/filecache_test.go

@@ -0,0 +1,171 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/spf13/afero"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileCache(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.concurrent]
+ttl = 111
+dir = "/cache/c"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	caches, err := NewCachesFromConfig(afero.NewMemMapFs(), cfg)
+	assert.NoError(err)
+
+	const cacheName = "Concurrent"
+
+	c := caches.Get(cacheName)
+	assert.NotNil(c)
+	assert.Equal(111, c.ttl)
+
+	assert.NoError(c.WriteReader("a", strings.NewReader("abc")))
+
+	r := c.Get("a")
+	assert.NotNil(r)
+	b, _ := ioutil.ReadAll(r)
+	r.Close()
+	assert.Equal("abc", string(b))
+
+	assert.NoError(c.WriteFunc("b", func() (io.Reader, error) {
+		return strings.NewReader("bcd"), nil
+	}))
+
+	r = c.Get("b")
+	assert.NotNil(r)
+	b, _ = ioutil.ReadAll(r)
+	r.Close()
+	assert.Equal("bcd", string(b))
+
+	assert.NotNil(caches.Get(strings.ToUpper(cacheName)))
+
+}
+
+func TestFileCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.concurrent]
+ttl = 111
+dir = "/cache/c"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	caches, err := NewCachesFromConfig(afero.NewMemMapFs(), cfg)
+	assert.NoError(err)
+
+	const cacheName = "concurrent"
+
+	filenameData := func(i int) (string, string) {
+		data := fmt.Sprintf("data: %d", i)
+		filename := fmt.Sprintf("file%d", i)
+		return filename, data
+	}
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 10; j++ {
+				c := caches.Get(cacheName)
+				assert.NotNil(c)
+				filename, data := filenameData(i)
+				assert.NoError(c.WriteReader(filename, strings.NewReader(data)))
+			}
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 10; j++ {
+				c := caches.Get(cacheName)
+				assert.NotNil(c)
+				filename, data := filenameData(i)
+				r := c.Get(filename)
+				if r != nil {
+					b, _ := ioutil.ReadAll(r)
+					r.Close()
+					assert.Equal(data, string(b))
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func TestDecodeConfig(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.c1]
+ttl = 1234
+dir = "/path/to/c1"
+[caches.c2]
+ttl = 3456
+dir = "/path/to/c2"
+[caches.c3]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(afero.NewMemMapFs(), cfg)
+	assert.NoError(err)
+
+	assert.Equal(5, len(decoded))
+
+	c2 := decoded["c2"]
+	assert.Equal(3456, c2.TTL)
+	assert.Equal("/path/to/c2", c2.Dir)
+
+	c3 := decoded["c3"]
+	assert.Equal(-1, c3.TTL)
+	assert.Equal("/path/to/c3", c3.Dir)
+
+}