Add a file cache

Fixes gohugoio#5404

cache/filecache/filecache.go

@@ -0,0 +1,118 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"io"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/BurntSushi/locker"
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+)
+
+type Config struct {
+	// Time to Live. Any items older than this will be removed and
+	// not returned from the cache.
+	TTL int
+}
+
+// CI cache dir provider
+//
+
+// FileCache caches a set of files in a directory. This is usually a file on
+// disk, but since this is backed by an Afero file system, it can be anything.
+type FileCache struct {
+	fs afero.Fs
+
+	ttl time.Duration
+
+	nlocker *locker.Locker
+}
+
+// Get gets a file from the cache given a filename. It will return nil
+// if file was not found in cache or if it's expired.
+func (c *FileCache) Get(filename string) hugio.ReadSeekCloser {
+	filename = filepath.Clean(filename)
+	c.nlocker.RLock(filename)
+	defer c.nlocker.RUnlock(filename)
+
+	fi, err := c.fs.Stat(filename)
+	if err != nil {
+		return nil
+	}
+
+	expiry := time.Now().Add(-c.ttl)
+	if fi.ModTime().Before(expiry) {
+		c.fs.Remove(filename)
+		return nil
+	}
+
+	f, err := c.fs.Open(filename)
+	if err != nil {
+		return nil
+	}
+	return f
+}
+
+// WriteReader writes r to filename in the file cache.
+func (c *FileCache) WriteReader(filename string, r io.Reader) error {
+	filename = filepath.Clean(filename)
+	c.nlocker.Lock(filename)
+	defer c.nlocker.Unlock(filename)
+
+	return afero.WriteReader(c.fs, filename, r)
+}
+
+type FileCaches struct {
+	mu sync.Mutex
+
+	fs afero.Fs
+
+	m map[string]*FileCache
+}
+
+func (f *FileCaches) GetOrCreate(name string) *FileCache {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if c, ok := f.m[name]; ok {
+		return c
+	}
+
+	cfg := Config{TTL: 100}
+
+	c := &FileCache{
+		fs:      f.fs,
+		nlocker: locker.NewLocker(),
+		ttl:     time.Second * time.Duration(cfg.TTL),
+	}
+
+	f.m[name] = c
+
+	return c
+
+}
+
+// FileCaches holds a map of named file caches.
+//type FileCaches map[string]*FileCache
+
+// NewFileCachesFromConfig creates a new set of file caches from the given
+// configuration.
+func NewFileCachesFromConfig(fs afero.Fs, cfg config.Provider) *FileCaches {
+	return &FileCaches{fs: fs, m: make(map[string]*FileCache)}
+}

cache/filecache/filecache_test.go

@@ -0,0 +1,95 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/viper"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileCache(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	caches := NewFileCachesFromConfig(afero.NewMemMapFs(), viper.New())
+
+	const cacheName = "concurrent"
+
+	c := caches.GetOrCreate(cacheName)
+
+	data := "abc"
+
+	assert.NoError(c.WriteReader("a", strings.NewReader(data)))
+
+	r := c.Get("a")
+	assert.NotNil(r)
+	b, _ := ioutil.ReadAll(r)
+	r.Close()
+	assert.Equal(data, string(b))
+
+}
+
+func TestFileCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	caches := NewFileCachesFromConfig(afero.NewMemMapFs(), viper.New())
+
+	const cacheName = "concurrent"
+
+	filenameData := func(i int) (string, string) {
+		data := fmt.Sprintf("data: %d", i)
+		filename := fmt.Sprintf("file%d", i)
+		return filename, data
+	}
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 10; j++ {
+				c := caches.GetOrCreate(cacheName)
+				filename, data := filenameData(i)
+				assert.NoError(c.WriteReader(filename, strings.NewReader(data)))
+			}
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 10; j++ {
+				c := caches.GetOrCreate(cacheName)
+				filename, data := filenameData(i)
+				r := c.Get(filename)
+				if r != nil {
+					b, _ := ioutil.ReadAll(r)
+					r.Close()
+					assert.Equal(data, string(b))
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}