From e5f229974166402f51e4ee0695ffb4d1e09fa174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 25 Jul 2019 00:12:40 +0200 Subject: [PATCH] Block symlink dir traversal for /static This is in line with how it behaved before, but it was lifted a little for the project mount for Hugo Modules, but that could create hard-to-detect loops. --- cache/filecache/filecache_test.go | 2 +- deps/deps.go | 4 +- helpers/path_test.go | 6 +- helpers/pathspec.go | 9 +- helpers/pathspec_test.go | 2 +- helpers/testhelpers_test.go | 2 +- helpers/url_test.go | 8 +- hugofs/decorators.go | 25 +++-- hugofs/fileinfo.go | 19 ++++ hugofs/nosymlink_fs.go | 89 ++++++++++++++-- hugofs/nosymlink_test.go | 159 +++++++++++++++++++---------- hugofs/rootmapping_fs.go | 6 +- hugofs/walk.go | 23 ++++- hugolib/data/hugo.toml | 1 - hugolib/filesystems/basefs.go | 36 +++++-- hugolib/filesystems/basefs_test.go | 10 +- hugolib/hugo_modules_test.go | 32 ++++-- hugolib/hugo_sites_build_test.go | 3 +- hugolib/pages_capture_test.go | 2 +- resources/page/testhelpers_test.go | 2 +- resources/testhelpers_test.go | 4 +- source/content_directory_test.go | 2 +- source/filesystem_test.go | 2 +- tpl/data/resources_test.go | 2 +- 24 files changed, 320 insertions(+), 130 deletions(-) delete mode 100755 hugolib/data/hugo.toml diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index a03c3116a61..78becd43bd8 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -292,7 +292,7 @@ func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec cfg, err := config.FromConfigString(configStr, "toml") assert.NoError(err) initConfig(fs, cfg) - p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg) + p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, nil) assert.NoError(err) return p diff --git a/deps/deps.go b/deps/deps.go index 8ef015ac95b..aaed900e58a 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -207,7 +207,7 @@ func New(cfg DepsCfg) (*Deps, error) { cfg.OutputFormats = output.DefaultFormats } - ps, err := helpers.NewPathSpec(fs, cfg.Language) + ps, err := helpers.NewPathSpec(fs, cfg.Language, logger) if err != nil { return nil, errors.Wrap(err, "create PathSpec") @@ -272,7 +272,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er l := cfg.Language var err error - d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs) + d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.Log, d.BaseFs) if err != nil { return nil, err } diff --git a/helpers/path_test.go b/helpers/path_test.go index e58a045c19e..d27d2e9b925 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -60,7 +60,7 @@ func TestMakePath(t *testing.T) { v.Set("removePathAccents", test.removeAccents) l := langs.NewDefaultLanguage(v) - p, err := NewPathSpec(hugofs.NewMem(v), l) + p, err := NewPathSpec(hugofs.NewMem(v), l, nil) require.NoError(t, err) output := p.MakePath(test.input) @@ -73,7 +73,7 @@ func TestMakePath(t *testing.T) { func TestMakePathSanitized(t *testing.T) { v := newTestCfg() - p, _ := NewPathSpec(hugofs.NewMem(v), v) + p, _ := NewPathSpec(hugofs.NewMem(v), v, nil) tests := []struct { input string @@ -101,7 +101,7 @@ func TestMakePathSanitizedDisablePathToLower(t *testing.T) { v.Set("disablePathToLower", true) l := langs.NewDefaultLanguage(v) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, _ := NewPathSpec(hugofs.NewMem(v), l, nil) tests := []struct { input string diff --git a/helpers/pathspec.go b/helpers/pathspec.go index b82ebd99226..d61757b3d67 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -16,6 +16,7 @@ package helpers import ( "strings" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/filesystems" @@ -37,13 +38,13 @@ type PathSpec struct { } // NewPathSpec creats a new PathSpec from the given filesystems and language. -func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { - return NewPathSpecWithBaseBaseFsProvided(fs, cfg, nil) +func NewPathSpec(fs *hugofs.Fs, cfg config.Provider, logger *loggers.Logger) (*PathSpec, error) { + return NewPathSpecWithBaseBaseFsProvided(fs, cfg, logger, nil) } // NewPathSpecWithBaseBaseFsProvided creats a new PathSpec from the given filesystems and language. // If an existing BaseFs is provided, parts of that is reused. -func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.Provider, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) { +func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.Provider, logger *loggers.Logger, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) { p, err := paths.New(fs, cfg) if err != nil { @@ -56,7 +57,7 @@ func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.Provider, baseB filesystems.WithBaseFs(baseBaseFs), } } - bfs, err := filesystems.NewBase(p, options...) + bfs, err := filesystems.NewBase(p, logger, options...) if err != nil { return nil, err } diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index 1c27f7e1159..06a5a619904 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -42,7 +42,7 @@ func TestNewPathSpecFromConfig(t *testing.T) { fs := hugofs.NewMem(v) fs.Source.MkdirAll(filepath.FromSlash("thework/thethemes/thetheme"), 0777) - p, err := NewPathSpec(fs, l) + p, err := NewPathSpec(fs, l, nil) require.NoError(t, err) require.True(t, p.CanonifyURLs) diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index b74dccfc46d..2d12289c61e 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -10,7 +10,7 @@ import ( func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { l := langs.NewDefaultLanguage(v) - ps, _ := NewPathSpec(fs, l) + ps, _ := NewPathSpec(fs, l, nil) return ps } diff --git a/helpers/url_test.go b/helpers/url_test.go index a2c945dfe14..e049a1a0ca0 100644 --- a/helpers/url_test.go +++ b/helpers/url_test.go @@ -28,7 +28,7 @@ func TestURLize(t *testing.T) { v := newTestCfg() l := langs.NewDefaultLanguage(v) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, _ := NewPathSpec(hugofs.NewMem(v), l, nil) tests := []struct { input string @@ -90,7 +90,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, v.Set("baseURL", test.baseURL) v.Set("contentDir", "content") l := langs.NewLanguage(lang, v) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, _ := NewPathSpec(hugofs.NewMem(v), l, nil) output := p.AbsURL(test.input, addLanguage) expected := test.expected @@ -168,7 +168,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, v.Set("baseURL", test.baseURL) v.Set("canonifyURLs", test.canonify) l := langs.NewLanguage(lang, v) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, _ := NewPathSpec(hugofs.NewMem(v), l, nil) output := p.RelURL(test.input, addLanguage) @@ -256,7 +256,7 @@ func TestURLPrep(t *testing.T) { v := newTestCfg() v.Set("uglyURLs", d.ugly) l := langs.NewDefaultLanguage(v) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, _ := NewPathSpec(hugofs.NewMem(v), l, nil) output := p.URLPrep(d.input) if d.output != output { diff --git a/hugofs/decorators.go b/hugofs/decorators.go index 0a2b3971270..e93f53aabc2 100644 --- a/hugofs/decorators.go +++ b/hugofs/decorators.go @@ -90,19 +90,14 @@ func NewBaseFileDecorator(fs afero.Fs) afero.Fs { isSymlink := isSymlink(fi) if isSymlink { meta[metaKeyOriginalFilename] = filename - link, err := filepath.EvalSymlinks(filename) + var link string + var err error + link, fi, err = evalSymlinks(fs, filename) if err != nil { return nil, err } - - fi, err = fs.Stat(link) - if err != nil { - return nil, err - } - filename = link meta[metaKeyIsSymlink] = true - } opener := func() (afero.File, error) { @@ -117,6 +112,20 @@ func NewBaseFileDecorator(fs afero.Fs) afero.Fs { return ffs } +func evalSymlinks(fs afero.Fs, filename string) (string, os.FileInfo, error) { + link, err := filepath.EvalSymlinks(filename) + if err != nil { + return "", nil, err + } + + fi, err := fs.Stat(link) + if err != nil { + return "", nil, err + } + + return link, fi, nil +} + type baseFileDecoratorFs struct { afero.Fs decorate func(fi os.FileInfo, filename string) (os.FileInfo, error) diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index a2f12c429dd..5a0fc23630f 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -180,9 +180,20 @@ type FileMetaInfo interface { type fileInfoMeta struct { os.FileInfo + m FileMeta } +// Name returns the file's name. Note that we follow symlinks, +// if supported by the file system, and the Name given here will be the +// name of the symlink, which is what Hugo needs in all situations. +func (fi *fileInfoMeta) Name() string { + if name := fi.m.Name(); name != "" { + return name + } + return fi.FileInfo.Name() +} + func (fi *fileInfoMeta) Meta() FileMeta { return fi.m } @@ -295,3 +306,11 @@ func normalizeFilename(filename string) string { } return filename } + +func fileInfosToNames(fis []os.FileInfo) []string { + names := make([]string, len(fis)) + for i, d := range fis { + names[i] = d.Name() + } + return names +} diff --git a/hugofs/nosymlink_fs.go b/hugofs/nosymlink_fs.go index 42ab94b5ce7..409b6f03dc6 100644 --- a/hugofs/nosymlink_fs.go +++ b/hugofs/nosymlink_fs.go @@ -16,6 +16,9 @@ package hugofs import ( "errors" "os" + "path/filepath" + + "github.com/gohugoio/hugo/common/loggers" "github.com/spf13/afero" ) @@ -24,15 +27,48 @@ var ( ErrPermissionSymlink = errors.New("symlinks not allowed in this filesystem") ) -func NewNoSymlinkFs(fs afero.Fs) afero.Fs { - return &noSymlinkFs{Fs: fs} +// NewNoSymlinkFs creates a new filesystem that prevents symlinks. +func NewNoSymlinkFs(fs afero.Fs, logger *loggers.Logger, allowFiles bool) afero.Fs { + return &noSymlinkFs{Fs: fs, logger: logger, allowFiles: allowFiles} } // noSymlinkFs is a filesystem that prevents symlinking. type noSymlinkFs struct { + allowFiles bool // block dirs only + logger *loggers.Logger afero.Fs } +type noSymlinkFile struct { + fs *noSymlinkFs + afero.File +} + +func (f *noSymlinkFile) Readdir(count int) ([]os.FileInfo, error) { + fis, err := f.File.Readdir(count) + + filtered := fis[:0] + for _, x := range fis { + filename := filepath.Join(f.Name(), x.Name()) + if _, err := f.fs.checkSymlinkStatus(filename, x); err != nil { + // Log a warning and drop the file from the list + logUnsupportedSymlink(filename, f.fs.logger) + } else { + filtered = append(filtered, x) + } + } + + return filtered, err +} + +func (f *noSymlinkFile) Readdirnames(count int) ([]string, error) { + dirs, err := f.Readdir(count) + if err != nil { + return nil, err + } + return fileInfosToNames(dirs), nil +} + func (fs *noSymlinkFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { return fs.stat(name) } @@ -53,33 +89,68 @@ func (fs *noSymlinkFs) stat(name string) (os.FileInfo, bool, error) { if lstater, ok := fs.Fs.(afero.Lstater); ok { fi, wasLstat, err = lstater.LstatIfPossible(name) } else { - fi, err = fs.Fs.Stat(name) } + if err != nil { + return nil, false, err + } + + fi, err = fs.checkSymlinkStatus(name, fi) + + return fi, wasLstat, err +} + +func (fs *noSymlinkFs) checkSymlinkStatus(name string, fi os.FileInfo) (os.FileInfo, error) { var metaIsSymlink bool if fim, ok := fi.(FileMetaInfo); ok { - metaIsSymlink = fim.Meta().IsSymlink() + meta := fim.Meta() + metaIsSymlink = meta.IsSymlink() } - if metaIsSymlink || isSymlink(fi) { - return nil, wasLstat, ErrPermissionSymlink + if metaIsSymlink { + if fs.allowFiles && !fi.IsDir() { + return fi, nil + } + return nil, ErrPermissionSymlink } - return fi, wasLstat, err + // Also support non-decorated filesystems, e.g. the Os fs. + if isSymlink(fi) { + // Need to determine if this is a directory or not. + _, sfi, err := evalSymlinks(fs.Fs, name) + if err != nil { + return nil, err + } + if fs.allowFiles && !sfi.IsDir() { + // Return the original FileInfo to get the expected Name. + return fi, nil + } + return nil, ErrPermissionSymlink + } + + return fi, nil } func (fs *noSymlinkFs) Open(name string) (afero.File, error) { if _, _, err := fs.stat(name); err != nil { return nil, err } - return fs.Fs.Open(name) + return fs.wrapFile(fs.Fs.Open(name)) } func (fs *noSymlinkFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { if _, _, err := fs.stat(name); err != nil { return nil, err } - return fs.Fs.OpenFile(name, flag, perm) + return fs.wrapFile(fs.Fs.OpenFile(name, flag, perm)) +} + +func (fs *noSymlinkFs) wrapFile(f afero.File, err error) (afero.File, error) { + if err != nil { + return nil, err + } + + return &noSymlinkFile{File: f, fs: fs}, nil } diff --git a/hugofs/nosymlink_test.go b/hugofs/nosymlink_test.go index 6d0b99dccbb..5e1964419e1 100644 --- a/hugofs/nosymlink_test.go +++ b/hugofs/nosymlink_test.go @@ -18,6 +18,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/htesting" "github.com/spf13/afero" @@ -25,73 +27,120 @@ import ( "github.com/stretchr/testify/require" ) -func TestNoSymlinkFs(t *testing.T) { - if skipSymlink() { - t.Skip("Skip; os.Symlink needs administrator rights on Windows") - } +func prepareSymlinks(t *testing.T) (string, func()) { assert := require.New(t) - workDir, clean, err := htesting.CreateTempDir(Os, "hugo-nosymlink") + + workDir, clean, err := htesting.CreateTempDir(Os, "hugo-symlink-test") assert.NoError(err) - defer clean() wd, _ := os.Getwd() - defer func() { - os.Chdir(wd) - }() blogDir := filepath.Join(workDir, "blog") - blogFile := filepath.Join(blogDir, "a.txt") - assert.NoError(os.MkdirAll(blogDir, 0777)) - afero.WriteFile(Os, filepath.Join(blogFile), []byte("content"), 0777) + blogSubDir := filepath.Join(blogDir, "sub") + assert.NoError(os.MkdirAll(blogSubDir, 0777)) + blogFile1 := filepath.Join(blogDir, "a.txt") + blogFile2 := filepath.Join(blogSubDir, "b.txt") + afero.WriteFile(Os, filepath.Join(blogFile1), []byte("content1"), 0777) + afero.WriteFile(Os, filepath.Join(blogFile2), []byte("content2"), 0777) os.Chdir(workDir) assert.NoError(os.Symlink("blog", "symlinkdedir")) os.Chdir(blogDir) + assert.NoError(os.Symlink("sub", "symsub")) assert.NoError(os.Symlink("a.txt", "symlinkdedfile.txt")) - fs := NewNoSymlinkFs(Os) - ls := fs.(afero.Lstater) - symlinkedDir := filepath.Join(workDir, "symlinkdedir") - symlinkedFile := filepath.Join(blogDir, "symlinkdedfile.txt") - - // Check Stat and Lstat - for _, stat := range []func(name string) (os.FileInfo, error){ - func(name string) (os.FileInfo, error) { - return fs.Stat(name) - }, - func(name string) (os.FileInfo, error) { - fi, _, err := ls.LstatIfPossible(name) - return fi, err - }, - } { - _, err = stat(symlinkedDir) - assert.Equal(ErrPermissionSymlink, err) - _, err = stat(symlinkedFile) - assert.Equal(ErrPermissionSymlink, err) - - fi, err := stat(filepath.Join(workDir, "blog")) - assert.NoError(err) - assert.NotNil(fi) - - fi, err = stat(blogFile) - assert.NoError(err) - assert.NotNil(fi) + return workDir, func() { + clean() + os.Chdir(wd) } +} - // Check Open - _, err = fs.Open(symlinkedDir) - assert.Equal(ErrPermissionSymlink, err) - _, err = fs.OpenFile(symlinkedDir, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) - assert.Equal(ErrPermissionSymlink, err) - _, err = fs.OpenFile(symlinkedFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) - assert.Equal(ErrPermissionSymlink, err) - _, err = fs.Open(symlinkedFile) - assert.Equal(ErrPermissionSymlink, err) - f, err := fs.Open(blogDir) - assert.NoError(err) - f.Close() - f, err = fs.Open(blogFile) - assert.NoError(err) - f.Close() +func TestNoSymlinkFs(t *testing.T) { + if skipSymlink() { + t.Skip("Skip; os.Symlink needs administrator rights on Windows") + } + assert := require.New(t) + workDir, clean := prepareSymlinks(t) + defer clean() + + blogDir := filepath.Join(workDir, "blog") + blogFile1 := filepath.Join(blogDir, "a.txt") + + logger := loggers.NewWarningLogger() + + for _, bfs := range []afero.Fs{NewBaseFileDecorator(Os), Os} { + for _, allowFiles := range []bool{false, true} { + logger.WarnCounter.Reset() + fs := NewNoSymlinkFs(bfs, logger, allowFiles) + ls := fs.(afero.Lstater) + symlinkedDir := filepath.Join(workDir, "symlinkdedir") + symlinkedFilename := "symlinkdedfile.txt" + symlinkedFile := filepath.Join(blogDir, symlinkedFilename) + + assertFileErr := func(err error) { + if allowFiles { + assert.NoError(err) + } else { + assert.Equal(ErrPermissionSymlink, err) + } + } - // os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + assertFileStat := func(name string, fi os.FileInfo, err error) { + t.Helper() + assertFileErr(err) + if err == nil { + assert.NotNil(fi) + assert.Equal(name, fi.Name()) + } + } + + // Check Stat and Lstat + for _, stat := range []func(name string) (os.FileInfo, error){ + func(name string) (os.FileInfo, error) { + return fs.Stat(name) + }, + func(name string) (os.FileInfo, error) { + fi, _, err := ls.LstatIfPossible(name) + return fi, err + }, + } { + fi, err := stat(symlinkedDir) + assert.Equal(ErrPermissionSymlink, err) + fi, err = stat(symlinkedFile) + assertFileStat(symlinkedFilename, fi, err) + + fi, err = stat(filepath.Join(workDir, "blog")) + assert.NoError(err) + assert.NotNil(fi) + + fi, err = stat(blogFile1) + assert.NoError(err) + assert.NotNil(fi) + } + + // Check Open + _, err := fs.Open(symlinkedDir) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.OpenFile(symlinkedDir, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.OpenFile(symlinkedFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + assertFileErr(err) + _, err = fs.Open(symlinkedFile) + assertFileErr(err) + f, err := fs.Open(blogDir) + assert.NoError(err) + f.Close() + f, err = fs.Open(blogFile1) + assert.NoError(err) + f.Close() + + // Check readdir + f, err = fs.Open(workDir) + assert.NoError(err) + // There is at least one unsported symlink inside workDir + _, err = f.Readdir(-1) + f.Close() + assert.Equal(uint64(1), logger.WarnCounter.Count()) + + } + } } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index a1214a02c7f..31d78219da3 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -459,9 +459,5 @@ func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { if err != nil { return nil, err } - dirss := make([]string, len(dirs)) - for i, d := range dirs { - dirss[i] = d.Name() - } - return dirss, nil + return fileInfosToNames(dirs), nil } diff --git a/hugofs/walk.go b/hugofs/walk.go index eca74673746..6947660c8dc 100644 --- a/hugofs/walk.go +++ b/hugofs/walk.go @@ -121,8 +121,7 @@ func (w *Walkway) Walk() error { return nil } - if err == ErrPermissionSymlink { - w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", w.root) + if w.checkErr(w.root, err) { return nil } @@ -149,6 +148,19 @@ func lstatIfPossible(fs afero.Fs, path string) (os.FileInfo, bool, error) { return fi, false, err } +// checkErr returns true if the error is handled. +func (w *Walkway) checkErr(filename string, err error) bool { + if err == ErrPermissionSymlink { + logUnsupportedSymlink(filename, w.logger) + return true + } + return false +} + +func logUnsupportedSymlink(filename string, logger *loggers.Logger) { + logger.WARN.Printf("Unsupported symlink found in %q, skipping.", filename) +} + // walk recursively descends path, calling walkFn. // It follow symlinks if supported by the filesystem, but only the same path once. func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo, walkFn WalkFunc) error { @@ -168,16 +180,17 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo if dirEntries == nil { f, err := w.fs.Open(path) - if err != nil { + if w.checkErr(path, err) { + return nil + } return walkFn(path, info, errors.Wrapf(err, "walk: open %q (%q)", path, w.root)) } fis, err := f.Readdir(-1) f.Close() if err != nil { - if err == ErrPermissionSymlink { - w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", filename) + if w.checkErr(filename, err) { return nil } return walkFn(path, info, errors.Wrap(err, "walk: Readdir")) diff --git a/hugolib/data/hugo.toml b/hugolib/data/hugo.toml deleted file mode 100755 index eb1dbc42cb1..00000000000 --- a/hugolib/data/hugo.toml +++ /dev/null @@ -1 +0,0 @@ -slogan = "Hugo Rocks!" \ No newline at end of file diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 47ae5d0e030..d7dd8063029 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/hugofs/files" "github.com/pkg/errors" @@ -295,8 +297,11 @@ func WithBaseFs(b *BaseFs) func(*BaseFs) error { } // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase -func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { +func NewBase(p *paths.Paths, logger *loggers.Logger, options ...func(*BaseFs) error) (*BaseFs, error) { fs := p.Fs + if logger == nil { + logger = loggers.NewWarningLogger() + } publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) @@ -314,7 +319,7 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { return b, nil } - builder := newSourceFilesystemsBuilder(p, b) + builder := newSourceFilesystemsBuilder(p, logger, b) sourceFilesystems, err := builder.Build() if err != nil { return nil, errors.Wrap(err, "build filesystems") @@ -327,15 +332,16 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { } type sourceFilesystemsBuilder struct { + logger *loggers.Logger p *paths.Paths sourceFs afero.Fs result *SourceFilesystems theBigFs *filesystemsCollector } -func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder { +func newSourceFilesystemsBuilder(p *paths.Paths, logger *loggers.Logger, b *BaseFs) *sourceFilesystemsBuilder { sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source) - return &sourceFilesystemsBuilder{p: p, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} + return &sourceFilesystemsBuilder{p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} } func (b *sourceFilesystemsBuilder) newSourceFilesystem(fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { @@ -415,7 +421,7 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { ms[k] = sfs } } else { - bfs := afero.NewBasePathFs(b.theBigFs.overlayMounts, files.ComponentFolderStatic) + bfs := afero.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic) ms[""] = b.newSourceFilesystem(bfs, b.result.StaticDirs) } @@ -432,7 +438,7 @@ func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesys collector := &filesystemsCollector{ sourceProject: b.sourceFs, - sourceModules: hugofs.NewNoSymlinkFs(b.sourceFs), + sourceModules: hugofs.NewNoSymlinkFs(b.sourceFs, b.logger, false), overlayDirs: make(map[string][]hugofs.FileMetaInfo), staticPerLanguage: staticFsMap, } @@ -475,6 +481,10 @@ func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool { return strings.HasPrefix(mnt.Target, files.ComponentFolderContent) } +func (b *sourceFilesystemsBuilder) isStaticMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderStatic) +} + func (b *sourceFilesystemsBuilder) createModFs( collector *filesystemsCollector, md mountsDescriptor) error { @@ -482,6 +492,7 @@ func (b *sourceFilesystemsBuilder) createModFs( var ( fromTo []hugofs.RootMapping fromToContent []hugofs.RootMapping + fromToStatic []hugofs.RootMapping ) absPathify := func(path string) string { @@ -544,6 +555,8 @@ OUTER: if isContentMount { fromToContent = append(fromToContent, rm) + } else if b.isStaticMount(mount) { + fromToStatic = append(fromToStatic, rm) } else { fromTo = append(fromTo, rm) } @@ -553,6 +566,7 @@ OUTER: if !md.isMainProject { modBase = collector.sourceModules } + sourceStatic := hugofs.NewNoSymlinkFs(modBase, b.logger, true) rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...) if err != nil { @@ -562,17 +576,22 @@ OUTER: if err != nil { return err } + rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...) + if err != nil { + return err + } // We need to keep the ordered list of directories for watching and // some special merge operations (data, i18n). collector.addDirs(rmfs) collector.addDirs(rmfsContent) + collector.addDirs(rmfsStatic) if collector.staticPerLanguage != nil { for _, l := range b.p.Languages { lang := l.Lang - lfs := rmfs.Filter(func(rm hugofs.RootMapping) bool { + lfs := rmfsStatic.Filter(func(rm hugofs.RootMapping) bool { rlang := rm.Meta.Lang() return rlang == "" || rlang == lang }) @@ -599,12 +618,14 @@ OUTER: if collector.overlayMounts == nil { collector.overlayMounts = rmfs collector.overlayMountsContent = rmfsContent + collector.overlayMountsStatic = rmfsStatic collector.overlayFull = afero.NewBasePathFs(modBase, md.dir) collector.overlayResources = afero.NewBasePathFs(modBase, getResourcesDir()) } else { collector.overlayMounts = afero.NewCopyOnWriteFs(collector.overlayMounts, rmfs) collector.overlayMountsContent = hugofs.NewLanguageCompositeFs(collector.overlayMountsContent, rmfsContent) + collector.overlayMountsStatic = hugofs.NewLanguageCompositeFs(collector.overlayMountsStatic, rmfsStatic) collector.overlayFull = afero.NewCopyOnWriteFs(collector.overlayFull, afero.NewBasePathFs(modBase, md.dir)) collector.overlayResources = afero.NewCopyOnWriteFs(collector.overlayResources, afero.NewBasePathFs(modBase, getResourcesDir())) } @@ -639,6 +660,7 @@ type filesystemsCollector struct { overlayMounts afero.Fs overlayMountsContent afero.Fs + overlayMountsStatic afero.Fs overlayFull afero.Fs overlayResources afero.Fs diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index eccbe00f222..e4efa68e21c 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -124,7 +124,7 @@ theme = ["atheme"] p, err := paths.New(fs, v) assert.NoError(err) - bfs, err := NewBase(p) + bfs, err := NewBase(p, nil) assert.NoError(err) assert.NotNil(bfs) @@ -206,7 +206,7 @@ func TestNewBaseFsEmpty(t *testing.T) { p, err := paths.New(fs, v) assert.NoError(err) - bfs, err := NewBase(p) + bfs, err := NewBase(p, nil) assert.NoError(err) assert.NotNil(bfs) assert.NotNil(bfs.Archetypes.Fs) @@ -263,7 +263,7 @@ func TestRealDirs(t *testing.T) { p, err := paths.New(fs, v) assert.NoError(err) - bfs, err := NewBase(p) + bfs, err := NewBase(p, nil) assert.NoError(err) assert.NotNil(bfs) @@ -300,7 +300,7 @@ func TestStaticFs(t *testing.T) { p, err := paths.New(fs, v) assert.NoError(err) - bfs, err := NewBase(p) + bfs, err := NewBase(p, nil) assert.NoError(err) sfs := bfs.StaticFs("en") @@ -344,7 +344,7 @@ func TestStaticFsMultiHost(t *testing.T) { p, err := paths.New(fs, v) assert.NoError(err) - bfs, err := NewBase(p) + bfs, err := NewBase(p, nil) assert.NoError(err) enFs := bfs.StaticFs("en") checkFileContent(enFs, "f1.txt", assert, "Hugo Rocks!") diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index dc0da2e1cb9..171bbb347e5 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -443,6 +443,7 @@ weight = 2 ` b := newTestSitesBuilder(t).WithNothingAdded().WithWorkingDir(workDir) + b.WithLogger(loggers.NewErrorLogger()) b.Fs = fs b.WithConfigFile("toml", config) @@ -457,35 +458,46 @@ weight = 2 bfs := b.H.BaseFs - for _, componentFs := range []afero.Fs{ + for i, componentFs := range []afero.Fs{ + bfs.Static[""].Fs, bfs.Archetypes.Fs, bfs.Content.Fs, bfs.Data.Fs, bfs.Assets.Fs, - bfs.Static[""].Fs, bfs.I18n.Fs} { - for i, id := range []string{"mod", "project"} { + if i != 0 { + continue + } + + for j, id := range []string{"mod", "project"} { + + statCheck := func(fs afero.Fs, filename string, isDir bool) { + shouldFail := j == 0 + if !shouldFail && i == 0 { + // Static dirs only supports symlinks for files + shouldFail = isDir + } - statCheck := func(fs afero.Fs, filename string) { - shouldFail := i == 0 _, err := fs.Stat(filepath.FromSlash(filename)) + if err != nil { - if strings.HasSuffix(filename, "toml") && strings.Contains(err.Error(), "files not supported") { + if i > 0 && strings.HasSuffix(filename, "toml") && strings.Contains(err.Error(), "files not supported") { // OK return } } + if shouldFail { assert.Error(err) - assert.Equal(hugofs.ErrPermissionSymlink, err) + assert.Equal(hugofs.ErrPermissionSymlink, err, filename) } else { - assert.NoError(err) + assert.NoError(err, filename) } } - statCheck(componentFs, fmt.Sprintf("realsym%s", id)) - statCheck(componentFs, fmt.Sprintf("real/datasym%s.toml", id)) + statCheck(componentFs, fmt.Sprintf("realsym%s", id), true) + statCheck(componentFs, fmt.Sprintf("real/datasym%s.toml", id), false) } } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 6ff8ae4d2d5..876f21cfa6d 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -2,7 +2,6 @@ package hugolib import ( "fmt" - "os" "strings" "testing" @@ -1282,7 +1281,7 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { root = helpers.FilePathSeparator + root } - helpers.PrintFs(fs, root, os.Stdout) + //helpers.PrintFs(fs, root, os.Stdout) t.Fatalf("Failed to read file: %s", err) } return string(b) diff --git a/hugolib/pages_capture_test.go b/hugolib/pages_capture_test.go index f8a7833bea3..38391f85cda 100644 --- a/hugolib/pages_capture_test.go +++ b/hugolib/pages_capture_test.go @@ -52,7 +52,7 @@ func TestPagesCapture(t *testing.T) { writeFile("pages/page2.md") writeFile("pages/page.png") - ps, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg) + ps, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, loggers.NewErrorLogger()) assert.NoError(err) sourceSpec := source.NewSourceSpec(ps, fs) diff --git a/resources/page/testhelpers_test.go b/resources/page/testhelpers_test.go index fa5f8e9c8a0..1a27985576d 100644 --- a/resources/page/testhelpers_test.go +++ b/resources/page/testhelpers_test.go @@ -73,7 +73,7 @@ func newTestPathSpecFor(cfg config.Provider) *helpers.PathSpec { } cfg.Set("allModules", modules.Modules{mod}) fs := hugofs.NewMem(cfg) - s, err := helpers.NewPathSpec(fs, cfg) + s, err := helpers.NewPathSpec(fs, cfg, nil) if err != nil { panic(err) } diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index a2e726e16a8..1b2be00d703 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -66,7 +66,7 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * fs := hugofs.NewMem(cfg) - s, err := helpers.NewPathSpec(fs, cfg) + s, err := helpers.NewPathSpec(fs, cfg, nil) assert.NoError(err) filecaches, err := filecache.NewCaches(s) @@ -104,7 +104,7 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { fs.Destination = &afero.MemMapFs{} fs.Source = afero.NewBasePathFs(hugofs.Os, workDir) - s, err := helpers.NewPathSpec(fs, cfg) + s, err := helpers.NewPathSpec(fs, cfg, nil) assert.NoError(err) filecaches, err := filecache.NewCaches(s) diff --git a/source/content_directory_test.go b/source/content_directory_test.go index 7f050e0daa9..46fd7813ee1 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -54,7 +54,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { v := newTestConfig() v.Set("ignoreFiles", test.ignoreFilesRegexpes) fs := hugofs.NewMem(v) - ps, err := helpers.NewPathSpec(fs, v) + ps, err := helpers.NewPathSpec(fs, v, nil) assert.NoError(err) s := NewSourceSpec(ps, fs.Source) diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 33007c7e496..fd3ff195295 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -103,7 +103,7 @@ func newTestConfig() *viper.Viper { func newTestSourceSpec() *SourceSpec { v := newTestConfig() fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(afero.NewMemMapFs()), v) - ps, err := helpers.NewPathSpec(fs, v) + ps, err := helpers.NewPathSpec(fs, v, nil) if err != nil { panic(err) } diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index 39cf6bfa9fd..e9850c2264b 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -203,7 +203,7 @@ func newDeps(cfg config.Provider) *deps.Deps { fs := hugofs.NewMem(cfg) logger := loggers.NewErrorLogger() - p, err := helpers.NewPathSpec(fs, cfg) + p, err := helpers.NewPathSpec(fs, cfg, nil) if err != nil { panic(err) }