Add a file cache

Fixes gohugoio#5404

cache/filecache/filecache.go

@@ -0,0 +1,265 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/gohugoio/hugo/hugolib/paths"
+
+	"github.com/pkg/errors"
+
+	"github.com/BurntSushi/locker"
+	"github.com/bep/mapstructure"
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/spf13/afero"
+)
+
+const cachesConfigKey = "caches"
+
+var defaultCacheConfig = cacheConfig{
+	MaxAge: -1,
+	Dir:    ":cacheDir",
+}
+
+const (
+	cacheKeyGetJSON = "getjson"
+	cacheKeyGetCSV  = "getcsv"
+)
+
+var defaultCacheConfigs = map[string]cacheConfig{
+	cacheKeyGetJSON: defaultCacheConfig,
+	cacheKeyGetCSV:  defaultCacheConfig,
+}
+
+type cachesConfig map[string]cacheConfig
+
+type cacheConfig struct {
+	// Maxe age of ache entries in this cache. Any items older than this will
+	// be removed and not returned from the cache.
+	// -1 means forever, 0 means cache is disabled.
+	MaxAge int
+
+	// The directory where files are stored.
+	Dir string
+}
+
+// Cache caches a set of files in a directory. This is usually a file on
+// disk, but since this is backed by an Afero file system, it can be anything.
+type Cache struct {
+	fs afero.Fs
+
+	// Max age in seconds.
+	maxAge int
+
+	nlocker *locker.Locker
+}
+
+// NewCache creates a new file cache with the given filesystem and max age.
+func NewCache(fs afero.Fs, maxAge int) *Cache {
+	return &Cache{
+		fs:      fs,
+		nlocker: locker.NewLocker(),
+		maxAge:  maxAge,
+	}
+}
+
+// GetOrCreate tries to get the named file from cache. If not found or expired, f will
+// be invoked and the result cached.
+// This method is protected by a named lock using the given name as identifier.
+func (c *Cache) GetOrCreate(name string, f func() (io.ReadCloser, error)) (io.ReadCloser, error) {
+	name = filepath.Clean(name)
+	c.nlocker.RLock(name)
+	r, expired := c.get(name)
+	if !expired && r != nil {
+		c.nlocker.RUnlock(name)
+		return r, nil
+	}
+	c.nlocker.RUnlock(name)
+
+	// Need a write lock for the rest.
+	c.nlocker.Lock(name)
+	defer c.nlocker.Unlock(name)
+
+	// Double check.
+	r, expired = c.get(name)
+	if !expired && r != nil {
+		return r, nil
+	}
+
+	if expired {
+		c.fs.Remove(name)
+	}
+
+	nr, err := f()
+	if err != nil {
+		return nil, err
+	}
+
+	if c.maxAge == 0 {
+		// No caching.
+		return struct {
+			io.Reader
+			io.Closer
+		}{
+			nr,
+			ioutil.NopCloser(nil),
+		}, nil
+	}
+
+	var buff bytes.Buffer
+
+	return struct {
+		io.Reader
+		io.Closer
+	}{
+		&buff,
+		ioutil.NopCloser(nil),
+	}, afero.WriteReader(c.fs, name, io.TeeReader(nr, &buff))
+
+}
+
+func (c *Cache) get(name string) (hugio.ReadSeekCloser, bool) {
+	if c.maxAge == 0 {
+		return nil, false
+	}
+
+	if c.maxAge > 0 {
+		fi, err := c.fs.Stat(name)
+		if err != nil {
+			return nil, false
+		}
+
+		expiry := time.Now().Add(-time.Duration(c.maxAge) * time.Second)
+		expired := fi.ModTime().Before(expiry)
+		if expired {
+			return nil, true
+		}
+	}
+
+	f, err := c.fs.Open(name)
+	if err != nil {
+		return nil, false
+	}
+	return f, false
+}
+
+type Caches map[string]*Cache
+
+// Get gets a named cache, nil if none found.
+func (f Caches) Get(name string) *Cache {
+	return f[strings.ToLower(name)]
+}
+
+// NewCachesFromPaths creates a new set of file caches from the given
+// configuration.
+func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
+	dcfg, err := decodeConfig(p)
+	if err != nil {
+		return nil, err
+	}
+
+	fs := p.Fs.Source
+
+	m := make(Caches)
+	for k, v := range dcfg {
+		// TODO(bep) cache placeholders + CI?
+		baseDir := filepath.Join(v.Dir, k)
+		bfs := afero.NewBasePathFs(fs, baseDir)
+		m[k] = NewCache(bfs, v.MaxAge)
+	}
+
+	return m, nil
+}
+
+func decodeConfig(p *paths.Paths) (cachesConfig, error) {
+	c := make(cachesConfig)
+	// Add defaults
+	for k, v := range defaultCacheConfigs {
+		c[k] = v
+	}
+
+	cfg := p.Cfg
+
+	if !cfg.IsSet(cachesConfigKey) {
+		return c, nil
+	}
+
+	m := cfg.GetStringMap(cachesConfigKey)
+
+	for k, v := range m {
+		cc := defaultCacheConfig
+
+		if err := mapstructure.WeakDecode(v, &cc); err != nil {
+			return nil, err
+		}
+
+		if cc.Dir == "" {
+			return c, errors.New("must provide cache Dir")
+		}
+
+		c[strings.ToLower(k)] = cc
+	}
+
+	for k, v := range c {
+		v.Dir = filepath.Clean(v.Dir)
+		dir := filepath.ToSlash(v.Dir)
+		parts := strings.Split(dir, "/")
+		first := parts[0]
+
+		if !strings.HasPrefix(first, ":") {
+			// Then it must be an absolute path and we can use it as is.
+			if !filepath.IsAbs(v.Dir) {
+				return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir) or be an absolute path", v.Dir)
+			}
+
+		} else {
+			resolved, err := resolveDirPlaceholder(p, first)
+			if err != nil {
+				return c, err
+			}
+			v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...))
+		}
+
+		c[k] = v
+	}
+
+	return c, nil
+}
+
+// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
+func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) {
+	switch strings.ToLower(placeholder) {
+	case ":resourcedir":
+		return p.AbsResourcesDir, nil
+	case ":cachedir":
+		cacheDir := p.Cfg.GetString("cacheDir")
+		if cacheDir == "" {
+			var err error
+			cacheDir, err = afero.TempDir(p.Fs.Source, "hugo_cache", "")
+			if err != nil {
+				return "", err
+			}
+		}
+		return cacheDir, nil
+	}
+
+	return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
+}