From 6cadf86075eedbd3244f9f82599398b80d566781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 8 Nov 2018 10:24:13 +0100 Subject: [PATCH] Add a file cache Fixes #5404 --- cache/filecache/filecache.go | 173 ++++++++++++++++++++++++++++++ cache/filecache/filecache_test.go | 133 +++++++++++++++++++++++ 2 files changed, 306 insertions(+) create mode 100644 cache/filecache/filecache.go create mode 100644 cache/filecache/filecache_test.go diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go new file mode 100644 index 00000000000..bc69e1f3137 --- /dev/null +++ b/cache/filecache/filecache.go @@ -0,0 +1,173 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "io" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/BurntSushi/locker" + "github.com/bep/mapstructure" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +const cachesConfigKey = "caches" + +type cachesConfig map[string]cacheConfig + +type cacheConfig struct { + // Time to Live. Any items older than this will be removed and + // not returned from the cache. + // -1 means forever. + TTL int + + // The directory where files are stored. + Dir string +} + +// CI cache dir provider +// + +// FileCache caches a set of files in a directory. This is usually a file on +// disk, but since this is backed by an Afero file system, it can be anything. +type FileCache struct { + fs afero.Fs + + ttl time.Duration + + nlocker *locker.Locker +} + +// Get gets a file from the cache given a filename. It will return nil +// if file was not found in cache or if it's expired. +func (c *FileCache) Get(filename string) hugio.ReadSeekCloser { + filename = filepath.Clean(filename) + + c.nlocker.RLock(filename) + + fi, err := c.fs.Stat(filename) + if err != nil { + c.nlocker.RUnlock(filename) + return nil + } + + expiry := time.Now().Add(-c.ttl) + expired := fi.ModTime().Before(expiry) + + if expired { + // Need a write lock for this. + c.nlocker.RUnlock(filename) + c.nlocker.Lock(filename) + + // Double check + fi, err := c.fs.Stat(filename) + expired := err == nil && fi.ModTime().Before(expiry) + if expired { + c.fs.Remove(filename) + } + + c.nlocker.Unlock(filename) + + if err != nil || expired { + return nil + } + + c.nlocker.RLock(filename) + + } + + defer c.nlocker.RUnlock(filename) + + f, err := c.fs.Open(filename) + if err != nil { + return nil + } + return f +} + +// WriteReader writes r to filename in the file cache. +func (c *FileCache) WriteReader(filename string, r io.Reader) error { + filename = filepath.Clean(filename) + c.nlocker.Lock(filename) + defer c.nlocker.Unlock(filename) + + return afero.WriteReader(c.fs, filename, r) +} + +type FileCaches struct { + mu sync.Mutex + + fs afero.Fs + + m map[string]*FileCache +} + +func (f *FileCaches) GetOrCreate(name string) *FileCache { + f.mu.Lock() + defer f.mu.Unlock() + + if c, ok := f.m[name]; ok { + return c + } + + cfg := cacheConfig{TTL: 100} + + c := &FileCache{ + fs: f.fs, + nlocker: locker.NewLocker(), + ttl: time.Second * time.Duration(cfg.TTL), + } + + f.m[name] = c + + return c + +} + +// FileCaches holds a map of named file caches. +//type FileCaches map[string]*FileCache + +// NewFileCachesFromConfig creates a new set of file caches from the given +// configuration. +func NewFileCachesFromConfig(fs afero.Fs, cfg config.Provider) *FileCaches { + return &FileCaches{fs: fs, m: make(map[string]*FileCache)} +} + +func decodeConfig(cfg config.Provider) (cachesConfig, error) { + c := make(cachesConfig) + if !cfg.IsSet(cachesConfigKey) { + return c, nil + } + + m := cfg.GetStringMap(cachesConfigKey) + + for k, v := range m { + cc := cacheConfig{ + TTL: -1, + } + + if err := mapstructure.WeakDecode(v, &cc); err != nil { + return nil, err + } + + c[strings.ToLower(k)] = cc + } + + return c, nil +} diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go new file mode 100644 index 00000000000..3dff692f87d --- /dev/null +++ b/cache/filecache/filecache_test.go @@ -0,0 +1,133 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "fmt" + "io/ioutil" + "strings" + "sync" + "testing" + + "github.com/gohugoio/hugo/config" + + "github.com/spf13/afero" + "github.com/spf13/viper" + + "github.com/stretchr/testify/require" +) + +func TestFileCache(t *testing.T) { + t.Parallel() + assert := require.New(t) + + caches := NewFileCachesFromConfig(afero.NewMemMapFs(), viper.New()) + + const cacheName = "concurrent" + + c := caches.GetOrCreate(cacheName) + + data := "abc" + + assert.NoError(c.WriteReader("a", strings.NewReader(data))) + + r := c.Get("a") + assert.NotNil(r) + b, _ := ioutil.ReadAll(r) + r.Close() + assert.Equal(data, string(b)) + +} + +func TestFileCacheConcurrent(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + caches := NewFileCachesFromConfig(afero.NewMemMapFs(), viper.New()) + + const cacheName = "concurrent" + + filenameData := func(i int) (string, string) { + data := fmt.Sprintf("data: %d", i) + filename := fmt.Sprintf("file%d", i) + return filename, data + } + + var wg sync.WaitGroup + + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 10; j++ { + c := caches.GetOrCreate(cacheName) + filename, data := filenameData(i) + assert.NoError(c.WriteReader(filename, strings.NewReader(data))) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 10; j++ { + c := caches.GetOrCreate(cacheName) + filename, data := filenameData(i) + r := c.Get(filename) + if r != nil { + b, _ := ioutil.ReadAll(r) + r.Close() + assert.Equal(data, string(b)) + } + } + }() + } + wg.Wait() +} + +func TestDecodeConfig(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + configStr := ` +[caches] +[caches.c1] +ttl = 1234 +dir = "/path/to/c1" +[caches.c2] +ttl = 3456 +dir = "/path/to/c2" +[caches.c3] +dir = "/path/to/c3" + +` + + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + + decoded, err := decodeConfig(cfg) + assert.NoError(err) + + assert.Equal(3, len(decoded)) + + c2 := decoded["c2"] + assert.Equal(3456, c2.TTL) + assert.Equal("/path/to/c2", c2.Dir) + + c3 := decoded["c3"] + assert.Equal(-1, c3.TTL) + assert.Equal("/path/to/c3", c3.Dir) + +}