Skip to content

Commit

Permalink
hugofs: Add includeFiles and excludeFiles to mount configuration
Browse files Browse the repository at this point in the history
Fixes #9042
  • Loading branch information
bep committed Oct 20, 2021
1 parent 94a5bac commit 471ed91
Show file tree
Hide file tree
Showing 15 changed files with 795 additions and 131 deletions.
6 changes: 4 additions & 2 deletions create/content.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,9 @@ func (b *contentBuilder) buildDir() error {
if !b.dirMap.siteUsed {
// We don't need to build everything.
contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
filename = strings.TrimPrefix(filename, string(os.PathSeparator))
for _, cn := range contentTargetFilenames {
if strings.HasPrefix(cn, filename) {
if strings.Contains(cn, filename) {
return true
}
}
Expand Down Expand Up @@ -205,7 +206,8 @@ func (b *contentBuilder) buildFile() error {
if !usesSite {
// We don't need to build everything.
contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
return strings.HasPrefix(contentPlaceholderAbsFilename, filename)
filename = strings.TrimPrefix(filename, string(os.PathSeparator))
return strings.Contains(contentPlaceholderAbsFilename, filename)
})
}

Expand Down
12 changes: 12 additions & 0 deletions docs/content/en/hugo-modules/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,15 @@ target
lang
: The language code, e.g. "en". Only relevant for `content` mounts, and `static` mounts when in multihost mode.

includeFiles (string or slice)
: One or more [glob](https://github.com/gobwas/glob) patterns matching files or directories to include. If `excludeFiles` is not set, the files matching `includeFiles` will be the files mounted.

The glob patterns are matched to the filenames starting from the `source` root, they should have Unix styled slashes even on Windows, `/` matches the mount root and `**` can be used as a super-asterisk to match recursively down all directories, e.g `/posts/**.jpg`.

The search is case-insensitive.

{{< new-in "0.89.0" >}}

excludeFiles (string or slice)
: One or more glob patterns matching files to exclude.

12 changes: 12 additions & 0 deletions hugofs/fileinfo.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"strings"
"time"

"github.com/gohugoio/hugo/hugofs/glob"

"github.com/gohugoio/hugo/hugofs/files"
"golang.org/x/text/unicode/norm"

Expand Down Expand Up @@ -76,6 +78,9 @@ type FileMeta struct {
Fs afero.Fs
OpenFunc func() (afero.File, error)
JoinStatFunc func(name string) (FileMetaInfo, error)

// Include only files or directories that match.
InclusionFilter *glob.FilenameFilter
}

func (m *FileMeta) Copy() *FileMeta {
Expand All @@ -95,10 +100,17 @@ func (m *FileMeta) Merge(from *FileMeta) {

for i := 0; i < dstv.NumField(); i++ {
v := dstv.Field(i)
if !v.CanSet() {
continue
}
if !hreflect.IsTruthfulValue(v) {
v.Set(srcv.Field(i))
}
}

if m.InclusionFilter == nil {
m.InclusionFilter = from.InclusionFilter
}
}

func (f *FileMeta) Open() (afero.File, error) {
Expand Down
170 changes: 170 additions & 0 deletions hugofs/filename_filter_fs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugofs

import (
"os"
"strings"
"syscall"
"time"

"github.com/gohugoio/hugo/hugofs/glob"
"github.com/spf13/afero"
)

func newFilenameFilterFs(fs afero.Fs, base string, filter *glob.FilenameFilter) afero.Fs {
return &filenameFilterFs{
fs: fs,
base: base,
filter: filter,
}
}

// filenameFilterFs is a filesystem that filters by filename.
type filenameFilterFs struct {
base string
fs afero.Fs

filter *glob.FilenameFilter
}

func (fs *filenameFilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
fi, b, err := fs.fs.(afero.Lstater).LstatIfPossible(name)
if err != nil {
return nil, false, err
}
if !fs.filter.Match(name, fi.IsDir()) {
return nil, false, os.ErrNotExist
}
return fi, b, nil
}

func (fs *filenameFilterFs) Open(name string) (afero.File, error) {
fi, err := fs.fs.Stat(name)
if err != nil {
return nil, err
}

if !fs.filter.Match(name, fi.IsDir()) {
return nil, os.ErrNotExist
}

f, err := fs.fs.Open(name)
if err != nil {
return nil, err
}

if !fi.IsDir() {
return f, nil
}

return &filenameFilterDir{
File: f,
base: fs.base,
filter: fs.filter,
}, nil
}

func (fs *filenameFilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
return fs.Open(name)
}

func (fs *filenameFilterFs) Stat(name string) (os.FileInfo, error) {
fi, _, err := fs.LstatIfPossible(name)
return fi, err
}

func (fs *filenameFilterFs) getOpener(name string) func() (afero.File, error) {
return func() (afero.File, error) {
return fs.Open(name)
}
}

type filenameFilterDir struct {
afero.File
base string
filter *glob.FilenameFilter
}

func (f *filenameFilterDir) Readdir(count int) ([]os.FileInfo, error) {
fis, err := f.File.Readdir(-1)
if err != nil {
return nil, err
}

var result []os.FileInfo
for _, fi := range fis {
fim := fi.(FileMetaInfo)
if f.filter.Match(strings.TrimPrefix(fim.Meta().Filename, f.base), fim.IsDir()) {
result = append(result, fi)
}
}

return result, nil
}

func (f *filenameFilterDir) Readdirnames(count int) ([]string, error) {
dirsi, err := f.Readdir(count)
if err != nil {
return nil, err
}

dirs := make([]string, len(dirsi))
for i, d := range dirsi {
dirs[i] = d.Name()
}
return dirs, nil
}

func (fs *filenameFilterFs) Chmod(n string, m os.FileMode) error {
return syscall.EPERM
}

func (fs *filenameFilterFs) Chtimes(n string, a, m time.Time) error {
return syscall.EPERM
}

func (fs *filenameFilterFs) Chown(n string, uid, gid int) error {
return syscall.EPERM
}

func (fs *filenameFilterFs) ReadDir(name string) ([]os.FileInfo, error) {
panic("not implemented")
}

func (fs *filenameFilterFs) Remove(n string) error {
return syscall.EPERM
}

func (fs *filenameFilterFs) RemoveAll(p string) error {
return syscall.EPERM
}

func (fs *filenameFilterFs) Rename(o, n string) error {
return syscall.EPERM
}
func (fs *filenameFilterFs) Create(n string) (afero.File, error) {
return nil, syscall.EPERM
}
func (fs *filenameFilterFs) Name() string {
return "FinameFilterFS"
}

func (fs *filenameFilterFs) Mkdir(n string, p os.FileMode) error {
return syscall.EPERM
}

func (fs *filenameFilterFs) MkdirAll(n string, p os.FileMode) error {
return syscall.EPERM
}
83 changes: 83 additions & 0 deletions hugofs/filename_filter_fs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugofs

import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"

"github.com/gohugoio/hugo/hugofs/glob"

"github.com/spf13/afero"

qt "github.com/frankban/quicktest"
)

func TestFilenameFilterFs(t *testing.T) {
c := qt.New(t)

base := filepath.FromSlash("/mybase")

fs := NewBaseFileDecorator(afero.NewMemMapFs())

for _, letter := range []string{"a", "b", "c"} {
for i := 1; i <= 3; i++ {
c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.txt", i)), []byte("some text file for"+letter), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.json", i)), []byte("some json file for"+letter), 0755), qt.IsNil)
}
}

fs = afero.NewBasePathFs(fs, base)

filter, err := glob.NewFilenameFilter(nil, []string{"/b/**.txt"})
c.Assert(err, qt.IsNil)

fs = newFilenameFilterFs(fs, base, filter)

assertExists := func(filename string, shouldExist bool) {
filename = filepath.Clean(filename)
_, err1 := fs.Stat(filename)
f, err2 := fs.Open(filename)
if shouldExist {
c.Assert(err1, qt.IsNil)
c.Assert(err2, qt.IsNil)
defer f.Close()

} else {
for _, err := range []error{err1, err2} {
c.Assert(err, qt.Not(qt.IsNil))
c.Assert(errors.Is(err, os.ErrNotExist), qt.IsTrue)
}
}
}

assertExists("/a/my1.txt", true)
assertExists("/b/my1.txt", false)

dirB, err := fs.Open("/b")
defer dirB.Close()
c.Assert(err, qt.IsNil)
dirBEntries, err := dirB.Readdirnames(-1)
c.Assert(dirBEntries, qt.DeepEquals, []string{"my1.json", "my2.json", "my3.json"})

dirC, err := fs.Open("/c")
defer dirC.Close()
c.Assert(err, qt.IsNil)
dirCEntries, err := dirC.Readdirnames(-1)
c.Assert(dirCEntries, qt.DeepEquals, []string{"my1.json", "my1.txt", "my2.json", "my2.txt", "my3.json", "my3.txt"})

}
Loading

0 comments on commit 471ed91

Please sign in to comment.