Add a file cache

Fixes gohugoio#5404

cache/filecache/filecache.go

@@ -0,0 +1,173 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"io"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/BurntSushi/locker"
+	"github.com/bep/mapstructure"
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+)
+
+const cachesConfigKey = "caches"
+
+type cachesConfig map[string]cacheConfig
+
+type cacheConfig struct {
+	// Time to Live. Any items older than this will be removed and
+	// not returned from the cache.
+	// -1 means forever.
+	TTL int
+
+	// The directory where files are stored.
+	Dir string
+}
+
+// CI cache dir provider
+//
+
+// FileCache caches a set of files in a directory. This is usually a file on
+// disk, but since this is backed by an Afero file system, it can be anything.
+type FileCache struct {
+	fs afero.Fs
+
+	ttl time.Duration
+
+	nlocker *locker.Locker
+}
+
+// Get gets a file from the cache given a filename. It will return nil
+// if file was not found in cache or if it's expired.
+func (c *FileCache) Get(filename string) hugio.ReadSeekCloser {
+	filename = filepath.Clean(filename)
+
+	c.nlocker.RLock(filename)
+
+	fi, err := c.fs.Stat(filename)
+	if err != nil {
+		c.nlocker.RUnlock(filename)
+		return nil
+	}
+
+	expiry := time.Now().Add(-c.ttl)
+	expired := fi.ModTime().Before(expiry)
+
+	if expired {
+		// Need a write lock for this.
+		c.nlocker.RUnlock(filename)
+		c.nlocker.Lock(filename)
+
+		// Double check
+		fi, err := c.fs.Stat(filename)
+		expired := err == nil && fi.ModTime().Before(expiry)
+		if expired {
+			c.fs.Remove(filename)
+		}
+
+		c.nlocker.Unlock(filename)
+
+		if err != nil || expired {
+			return nil
+		}
+
+		c.nlocker.RLock(filename)
+
+	}
+
+	defer c.nlocker.RUnlock(filename)
+
+	f, err := c.fs.Open(filename)
+	if err != nil {
+		return nil
+	}
+	return f
+}
+
+// WriteReader writes r to filename in the file cache.
+func (c *FileCache) WriteReader(filename string, r io.Reader) error {
+	filename = filepath.Clean(filename)
+	c.nlocker.Lock(filename)
+	defer c.nlocker.Unlock(filename)
+
+	return afero.WriteReader(c.fs, filename, r)
+}
+
+type FileCaches struct {
+	mu sync.Mutex
+
+	fs afero.Fs
+
+	m map[string]*FileCache
+}
+
+func (f *FileCaches) GetOrCreate(name string) *FileCache {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if c, ok := f.m[name]; ok {
+		return c
+	}
+
+	cfg := cacheConfig{TTL: 100}
+
+	c := &FileCache{
+		fs:      f.fs,
+		nlocker: locker.NewLocker(),
+		ttl:     time.Second * time.Duration(cfg.TTL),
+	}
+
+	f.m[name] = c
+
+	return c
+
+}
+
+// FileCaches holds a map of named file caches.
+//type FileCaches map[string]*FileCache
+
+// NewFileCachesFromConfig creates a new set of file caches from the given
+// configuration.
+func NewFileCachesFromConfig(fs afero.Fs, cfg config.Provider) *FileCaches {
+	return &FileCaches{fs: fs, m: make(map[string]*FileCache)}
+}
+
+func decodeConfig(cfg config.Provider) (cachesConfig, error) {
+	c := make(cachesConfig)
+	if !cfg.IsSet(cachesConfigKey) {
+		return c, nil
+	}
+
+	m := cfg.GetStringMap(cachesConfigKey)
+
+	for k, v := range m {
+		cc := cacheConfig{
+			TTL: -1,
+		}
+
+		if err := mapstructure.WeakDecode(v, &cc); err != nil {
+			return nil, err
+		}
+
+		c[strings.ToLower(k)] = cc
+	}
+
+	return c, nil
+}

cache/filecache/filecache_test.go

@@ -0,0 +1,133 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/viper"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileCache(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	caches := NewFileCachesFromConfig(afero.NewMemMapFs(), viper.New())
+
+	const cacheName = "concurrent"
+
+	c := caches.GetOrCreate(cacheName)
+
+	data := "abc"
+
+	assert.NoError(c.WriteReader("a", strings.NewReader(data)))
+
+	r := c.Get("a")
+	assert.NotNil(r)
+	b, _ := ioutil.ReadAll(r)
+	r.Close()
+	assert.Equal(data, string(b))
+
+}
+
+func TestFileCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	caches := NewFileCachesFromConfig(afero.NewMemMapFs(), viper.New())
+
+	const cacheName = "concurrent"
+
+	filenameData := func(i int) (string, string) {
+		data := fmt.Sprintf("data: %d", i)
+		filename := fmt.Sprintf("file%d", i)
+		return filename, data
+	}
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 10; j++ {
+				c := caches.GetOrCreate(cacheName)
+				filename, data := filenameData(i)
+				assert.NoError(c.WriteReader(filename, strings.NewReader(data)))
+			}
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 10; j++ {
+				c := caches.GetOrCreate(cacheName)
+				filename, data := filenameData(i)
+				r := c.Get(filename)
+				if r != nil {
+					b, _ := ioutil.ReadAll(r)
+					r.Close()
+					assert.Equal(data, string(b))
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func TestDecodeConfig(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.c1]
+ttl = 1234
+dir = "/path/to/c1"
+[caches.c2]
+ttl = 3456
+dir = "/path/to/c2"
+[caches.c3]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(cfg)
+	assert.NoError(err)
+
+	assert.Equal(3, len(decoded))
+
+	c2 := decoded["c2"]
+	assert.Equal(3456, c2.TTL)
+	assert.Equal("/path/to/c2", c2.Dir)
+
+	c3 := decoded["c3"]
+	assert.Equal(-1, c3.TTL)
+	assert.Equal("/path/to/c3", c3.Dir)
+
+}