diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go new file mode 100644 index 00000000000..882f4062139 --- /dev/null +++ b/cache/filecache/filecache.go @@ -0,0 +1,227 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "bytes" + "io" + "io/ioutil" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/BurntSushi/locker" + "github.com/bep/mapstructure" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +const cachesConfigKey = "caches" + +var defaultCacheConfig = cacheConfig{ + TTL: -1, + Dir: ":cacheDir", +} + +var defaultCacheConfigs = map[string]cacheConfig{ + "getjson": defaultCacheConfig, + "getcsv": defaultCacheConfig, +} + +type cachesConfig map[string]cacheConfig + +type cacheConfig struct { + // Time to Live. Any items older than this will be removed and + // not returned from the cache. + // -1 means forever. + TTL int + + // The directory where files are stored. + Dir string +} + +// Cache caches a set of files in a directory. This is usually a file on +// disk, but since this is backed by an Afero file system, it can be anything. +type Cache struct { + fs afero.Fs + + // Time to live, in seconds + ttl int + + nlocker *locker.Locker +} + +// NewCache creates a new file cache with the given filesystem and TTL. +func NewCache(fs afero.Fs, ttl int) *Cache { + return &Cache{ + fs: fs, + nlocker: locker.NewLocker(), + ttl: ttl, + } +} + +// GetOrCreate tries to get the named file from cache. If not found or expired, f will +// be invoked and the result cached. +// This method is protected by a named lock using the given name as identifier. +func (c *Cache) GetOrCreate(name string, f func() (io.ReadCloser, error)) (io.ReadCloser, error) { + name = filepath.Clean(name) + c.nlocker.RLock(name) + r, expired := c.get(name) + if !expired && r != nil { + c.nlocker.RUnlock(name) + return r, nil + } + c.nlocker.RUnlock(name) + + // Need a write lock for the rest. + c.nlocker.Lock(name) + defer c.nlocker.Unlock(name) + + // Double check. + r, expired = c.get(name) + if !expired && r != nil { + return r, nil + } + + if expired { + c.fs.Remove(name) + } + + nr, err := f() + if err != nil { + return nil, err + } + + if c.ttl == 0 { + // No caching. + return struct { + io.Reader + io.Closer + }{ + nr, + ioutil.NopCloser(nil), + }, nil + } + + var buff bytes.Buffer + + return struct { + io.Reader + io.Closer + }{ + &buff, + ioutil.NopCloser(nil), + }, afero.WriteReader(c.fs, name, io.TeeReader(nr, &buff)) + +} + +func (c *Cache) get(name string) (hugio.ReadSeekCloser, bool) { + if c.ttl == 0 { + return nil, false + } + + if c.ttl > 0 { + fi, err := c.fs.Stat(name) + if err != nil { + return nil, false + } + + expiry := time.Now().Add(-time.Duration(c.ttl) * time.Second) + expired := fi.ModTime().Before(expiry) + if expired { + return nil, true + } + } + + f, err := c.fs.Open(name) + if err != nil { + return nil, false + } + return f, false +} + +type Caches map[string]*Cache + +// Get gets a named cache, nil if none found. +func (f Caches) Get(name string) *Cache { + return f[strings.ToLower(name)] +} + +// NewCachesFromConfig creates a new set of file caches from the given +// configuration. +func NewCachesFromConfig(fs afero.Fs, cfg config.Provider) (Caches, error) { + dcfg, err := decodeConfig(fs, cfg) + if err != nil { + return nil, err + } + + m := make(Caches) + for k, v := range dcfg { + // TODO(bep) cache placeholders + CI? + baseDir := filepath.Join(k, v.Dir) + bfs := afero.NewBasePathFs(fs, baseDir) + m[k] = NewCache(bfs, v.TTL) + } + + return m, nil +} + +func decodeConfig(fs afero.Fs, cfg config.Provider) (cachesConfig, error) { + c := make(cachesConfig) + // Add defaults + for k, v := range defaultCacheConfigs { + c[k] = v + } + + if !cfg.IsSet(cachesConfigKey) { + return c, nil + } + + m := cfg.GetStringMap(cachesConfigKey) + + for k, v := range m { + cc := defaultCacheConfig + + if err := mapstructure.WeakDecode(v, &cc); err != nil { + return nil, err + } + + if cc.Dir == "" { + return c, errors.New("must provide cache Dir") + } + + c[strings.ToLower(k)] = cc + } + + cacheDir := cfg.GetString("cacheDir") + if cacheDir == "" { + var err error + cacheDir, err = afero.TempDir(fs, "hugo_cache", "") + if err != nil { + return c, err + } + } + + // Expand dir variables + // TODO(bep) cache + for k, v := range c { + v.Dir = strings.Replace(v.Dir, ":cacheDir", cacheDir, 1) + c[k] = v + } + + return c, nil +} diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go new file mode 100644 index 00000000000..c02f5426c09 --- /dev/null +++ b/cache/filecache/filecache_test.go @@ -0,0 +1,169 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "fmt" + "io" + "io/ioutil" + "strings" + "sync" + "testing" + "time" + + "github.com/gohugoio/hugo/config" + + "github.com/spf13/afero" + + "github.com/stretchr/testify/require" +) + +func TestFileCache(t *testing.T) { + t.Parallel() + assert := require.New(t) + + configStr := ` +[caches] +[caches.concurrent] +ttl = 111 +dir = "/cache/c" + +` + + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + + caches, err := NewCachesFromConfig(afero.NewMemMapFs(), cfg) + assert.NoError(err) + + const cacheName = "Concurrent" + + c := caches.Get(cacheName) + assert.NotNil(c) + assert.Equal(111, c.ttl) + + r, err := c.GetOrCreate("a", func() (io.ReadCloser, error) { + return struct { + io.ReadSeeker + io.Closer + }{ + strings.NewReader("abc"), + ioutil.NopCloser(nil), + }, nil + }) + + assert.NoError(err) + assert.NotNil(r) + b, _ := ioutil.ReadAll(r) + r.Close() + assert.Equal("abc", string(b)) + + assert.NotNil(caches.Get(strings.ToUpper(cacheName))) + +} + +func TestFileCacheConcurrent(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + configStr := ` +[caches] +[caches.concurrent] +ttl = 1 +dir = "/cache/c" + +` + + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + + caches, err := NewCachesFromConfig(afero.NewMemMapFs(), cfg) + assert.NoError(err) + + const cacheName = "concurrent" + + filenameData := func(i int) (string, string) { + data := fmt.Sprintf("data: %d", i) + filename := fmt.Sprintf("file%d", i) + return filename, data + } + + var wg sync.WaitGroup + + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 20; j++ { + c := caches.Get(cacheName) + assert.NotNil(c) + filename, data := filenameData(j) + r, err := c.GetOrCreate(filename, func() (io.ReadCloser, error) { + return struct { + io.ReadSeeker + io.Closer + }{ + strings.NewReader(data), + ioutil.NopCloser(nil), + }, nil + }) + assert.NoError(err) + b, _ := ioutil.ReadAll(r) + r.Close() + assert.Equal(data, string(b)) + // Trigger som expiration. + time.Sleep(200 * time.Millisecond) + } + }() + + } + wg.Wait() +} + +func TestDecodeConfig(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + configStr := ` +[caches] +[caches.c1] +ttl = 1234 +dir = "/path/to/c1" +[caches.c2] +ttl = 3456 +dir = "/path/to/c2" +[caches.c3] +dir = "/path/to/c3" + +` + + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + + decoded, err := decodeConfig(afero.NewMemMapFs(), cfg) + assert.NoError(err) + + assert.Equal(5, len(decoded)) + + c2 := decoded["c2"] + assert.Equal(3456, c2.TTL) + assert.Equal("/path/to/c2", c2.Dir) + + c3 := decoded["c3"] + assert.Equal(-1, c3.TTL) + assert.Equal("/path/to/c3", c3.Dir) + +} diff --git a/deps/deps.go b/deps/deps.go index db59ad212fa..58586c38cf0 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -4,6 +4,9 @@ import ( "sync" "time" + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" @@ -53,6 +56,9 @@ type Deps struct { // The configuration to use Cfg config.Provider `json:"-"` + // The file cache to use. + FileCaches filecache.Caches + // The translation func to use Translate func(translationID string, args ...interface{}) string `json:"-"` @@ -206,6 +212,11 @@ func New(cfg DepsCfg) (*Deps, error) { distinctErrorLogger := helpers.NewDistinctLogger(logger.ERROR) + fileCaches, err := filecache.NewCachesFromConfig(fs.Source, cfg.Cfg) + if err != nil { + return nil, errors.WithMessage(err, "failed to create file caches") + } + d := &Deps{ Fs: fs, Log: logger, @@ -219,6 +230,7 @@ func New(cfg DepsCfg) (*Deps, error) { ResourceSpec: resourceSpec, Cfg: cfg.Language, Language: cfg.Language, + FileCaches: fileCaches, BuildStartListeners: &Listeners{}, Timeout: time.Duration(timeoutms) * time.Millisecond, globalErrHandler: &globalErrHandler{}, diff --git a/tpl/data/cache.go b/tpl/data/cache.go deleted file mode 100644 index 6c4033160c3..00000000000 --- a/tpl/data/cache.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package data - -import ( - "crypto/md5" - "encoding/hex" - "errors" - "sync" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "github.com/spf13/afero" -) - -var cacheMu sync.RWMutex - -// getCacheFileID returns the cache ID for a string. -func getCacheFileID(cfg config.Provider, id string) string { - hash := md5.Sum([]byte(id)) - return cfg.GetString("cacheDir") + hex.EncodeToString(hash[:]) -} - -// getCache returns the content for an ID from the file cache or an error. -// If the ID is not found, return nil,nil. -func getCache(id string, fs afero.Fs, cfg config.Provider, ignoreCache bool) ([]byte, error) { - if ignoreCache { - return nil, nil - } - - cacheMu.RLock() - defer cacheMu.RUnlock() - - fID := getCacheFileID(cfg, id) - isExists, err := helpers.Exists(fID, fs) - if err != nil { - return nil, err - } - if !isExists { - return nil, nil - } - - return afero.ReadFile(fs, fID) -} - -// writeCache writes bytes associated with an ID into the file cache. -func writeCache(id string, c []byte, fs afero.Fs, cfg config.Provider, ignoreCache bool) error { - if ignoreCache { - return nil - } - - cacheMu.Lock() - defer cacheMu.Unlock() - - fID := getCacheFileID(cfg, id) - f, err := fs.Create(fID) - if err != nil { - return errors.New("Error: " + err.Error() + ". Failed to create file: " + fID) - } - defer f.Close() - - n, err := f.Write(c) - if err != nil { - return errors.New("Error: " + err.Error() + ". Failed to write to file: " + fID) - } - if n == 0 { - return errors.New("No bytes written to file: " + fID) - } - return nil -} - -func deleteCache(id string, fs afero.Fs, cfg config.Provider) error { - return fs.Remove(getCacheFileID(cfg, id)) -} diff --git a/tpl/data/cache_test.go b/tpl/data/cache_test.go deleted file mode 100644 index 6057f032182..00000000000 --- a/tpl/data/cache_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package data - -import ( - "fmt" - "testing" - - "github.com/spf13/afero" - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" -) - -func TestCache(t *testing.T) { - t.Parallel() - - fs := new(afero.MemMapFs) - - for i, test := range []struct { - path string - content []byte - ignore bool - }{ - {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false}, - {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false}, - {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false}, - {"трям/трям", []byte(`T€st трям/трям Content 123`), false}, - {"은행", []byte(`T€st C은행ontent 123`), false}, - {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false}, - {"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true}, - } { - msg := fmt.Sprintf("Test #%d: %v", i, test) - - cfg := viper.New() - - c, err := getCache(test.path, fs, cfg, test.ignore) - assert.NoError(t, err, msg) - assert.Nil(t, c, msg) - - err = writeCache(test.path, test.content, fs, cfg, test.ignore) - assert.NoError(t, err, msg) - - c, err = getCache(test.path, fs, cfg, test.ignore) - assert.NoError(t, err, msg) - - if test.ignore { - assert.Nil(t, c, msg) - } else { - assert.Equal(t, string(test.content), string(c)) - } - } -} diff --git a/tpl/data/data.go b/tpl/data/data.go index cecce4b4574..c6fe3581a3a 100644 --- a/tpl/data/data.go +++ b/tpl/data/data.go @@ -22,15 +22,25 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/deps" _errors "github.com/pkg/errors" ) // New returns a new instance of the data-namespaced template functions. func New(deps *deps.Deps) *Namespace { + + cacheGetJSON := deps.FileCaches.Get("getJSON") + cacheGetCSV := deps.FileCaches.Get("getCSV") + if cacheGetCSV == nil || cacheGetJSON == nil { + panic("invalid cache setup") + } + return &Namespace{ - deps: deps, - client: http.DefaultClient, + deps: deps, + cacheGetCSV: cacheGetCSV, + cacheGetJSON: cacheGetJSON, + client: http.DefaultClient, } } @@ -38,6 +48,9 @@ func New(deps *deps.Deps) *Namespace { type Namespace struct { deps *deps.Deps + cacheGetJSON *filecache.Cache + cacheGetCSV *filecache.Cache + client *http.Client } @@ -52,7 +65,7 @@ func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err e var clearCacheSleep = func(i int, u string) { ns.deps.Log.INFO.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep) time.Sleep(resSleep) - deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) + // TODO(bep) cache deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) } for i := 0; i <= resRetries; i++ { @@ -66,7 +79,7 @@ func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err e req.Header.Add("Accept", "text/plain") var c []byte - c, err = ns.getResource(req) + c, err = ns.getResource(ns.cacheGetCSV, req) if err != nil { return nil, _errors.Wrapf(err, "failed to read CSV resource %q", url) } @@ -103,7 +116,7 @@ func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) { req.Header.Add("Accept", "application/json") var c []byte - c, err = ns.getResource(req) + c, err = ns.getResource(ns.cacheGetJSON, req) if err != nil { return nil, _errors.Wrapf(err, "failed to get getJSON resource %q", url) } @@ -112,7 +125,7 @@ func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) { ns.deps.Log.INFO.Printf("Cannot read JSON from resource %s: %s", url, err) ns.deps.Log.INFO.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep) time.Sleep(resSleep) - deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) + // TODO(bep) cache deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) continue } break diff --git a/tpl/data/init.go b/tpl/data/init.go index 3bdc02786fc..b800d5ca02c 100644 --- a/tpl/data/init.go +++ b/tpl/data/init.go @@ -22,6 +22,7 @@ const name = "data" func init() { f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { + ctx := New(d) ns := &internal.TemplateFuncsNamespace{ diff --git a/tpl/data/init_test.go b/tpl/data/init_test.go index 6bb689a95f4..c4751e8925e 100644 --- a/tpl/data/init_test.go +++ b/tpl/data/init_test.go @@ -16,8 +16,8 @@ package data import ( "testing" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/tpl/internal" + "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -25,8 +25,11 @@ func TestInit(t *testing.T) { var found bool var ns *internal.TemplateFuncsNamespace + v := viper.New() + v.Set("contentDir", "content") + for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns = nsf(&deps.Deps{}) + ns = nsf(newDeps(v)) if ns.Name == name { found = true break diff --git a/tpl/data/resources.go b/tpl/data/resources.go index 11c35f9d9b6..41abc01a754 100644 --- a/tpl/data/resources.go +++ b/tpl/data/resources.go @@ -15,12 +15,15 @@ package data import ( "fmt" + "io" "io/ioutil" "net/http" "path/filepath" "sync" "time" + "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" @@ -63,53 +66,35 @@ func (l *remoteLock) URLUnlock(url string) { } // getRemote loads the content of a remote file. This method is thread safe. -func getRemote(req *http.Request, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) { +func getRemote(cache *filecache.Cache, req *http.Request, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) { url := req.URL.String() + id := helpers.MD5String(url) - c, err := getCache(url, fs, cfg, cfg.GetBool("ignoreCache")) - if err != nil { - return nil, err - } - if c != nil { - return c, nil - } + r, err := cache.GetOrCreate(id, func() (io.ReadCloser, error) { + jww.INFO.Printf("Downloading: %s ...", url) + res, err := hc.Do(req) + if err != nil { + return nil, err + } - // avoid race condition with locks, block other goroutines if the current url is processing - remoteURLLock.URLLock(url) - defer func() { remoteURLLock.URLUnlock(url) }() + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, fmt.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode)) + } - // avoid multiple locks due to calling getCache twice - c, err = getCache(url, fs, cfg, cfg.GetBool("ignoreCache")) - if err != nil { - return nil, err - } - if c != nil { - return c, nil - } - - jww.INFO.Printf("Downloading: %s ...", url) - res, err := hc.Do(req) - if err != nil { - return nil, err - } - - if res.StatusCode < 200 || res.StatusCode > 299 { - return nil, fmt.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode)) - } + return res.Body, nil + }) - c, err = ioutil.ReadAll(res.Body) - res.Body.Close() if err != nil { return nil, err } - err = writeCache(url, c, fs, cfg, cfg.GetBool("ignoreCache")) + b, err := ioutil.ReadAll(r) if err != nil { return nil, err } + r.Close() + return b, nil - jww.INFO.Printf("... and cached to: %s", getCacheFileID(cfg, url)) - return c, nil } // getLocal loads the content of a local file @@ -124,11 +109,11 @@ func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) { } // getResource loads the content of a local or remote file -func (ns *Namespace) getResource(req *http.Request) ([]byte, error) { +func (ns *Namespace) getResource(cache *filecache.Cache, req *http.Request) ([]byte, error) { switch req.URL.Scheme { case "": return getLocal(req.URL.String(), ns.deps.Fs.Source, ns.deps.Cfg) default: - return getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, ns.client) + return getRemote(cache, req, ns.deps.Fs.Source, ns.deps.Cfg, ns.client) } } diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index c1da36d055b..5a4bb297965 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" @@ -85,16 +86,16 @@ func getTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httpt func TestScpGetRemote(t *testing.T) { t.Parallel() fs := new(afero.MemMapFs) + cache := filecache.NewCache(fs, 100) tests := []struct { path string content []byte - ignore bool }{ - {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false}, - {"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`), false}, - {"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`), false}, - {"http://Doppel.Gänger/Fizz_Bazz-Bar", []byte(`T€st Банковский кассир Cont€nt 456`), true}, + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)}, + {"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`)}, + {"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`)}, + {"http://Doppel.Gänger/Fizz_Bazz-Bar", []byte(`T€st Банковский кассир Cont€nt 456`)}, } for _, test := range tests { @@ -110,19 +111,12 @@ func TestScpGetRemote(t *testing.T) { cfg := viper.New() - c, err := getRemote(req, fs, cfg, cl) + c, err := getRemote(cache, req, fs, cfg, cl) require.NoError(t, err, msg) assert.Equal(t, string(test.content), string(c)) - c, err = getCache(req.URL.String(), fs, cfg, test.ignore) - require.NoError(t, err, msg) - - if test.ignore { - assert.Empty(t, c, msg) - } else { - assert.Equal(t, string(test.content), string(c)) + assert.Equal(t, string(test.content), string(c)) - } } } @@ -152,7 +146,7 @@ func TestScpGetRemoteParallel(t *testing.T) { go func(gor int) { defer wg.Done() for j := 0; j < 10; j++ { - c, err := getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, cl) + c, err := getRemote(ns.cacheGetJSON, req, ns.deps.Fs.Source, ns.deps.Cfg, cl) assert.NoError(t, err) assert.Equal(t, string(content), string(c)) @@ -173,11 +167,15 @@ func newDeps(cfg config.Provider) *deps.Deps { panic(err) } + fs := hugofs.NewMem(l) logger := loggers.NewErrorLogger() + fileCaches, _ := filecache.NewCachesFromConfig(fs.Source, cfg) + return &deps.Deps{ Cfg: cfg, - Fs: hugofs.NewMem(l), + Fs: fs, + FileCaches: fileCaches, ContentSpec: cs, Log: logger, DistinctErrorLog: helpers.NewDistinctLogger(logger.ERROR),