diff --git a/commands/commands.go b/commands/commands.go index 1135dc01abb..6aacc8e0bde 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -204,6 +204,7 @@ type hugoBuilderCommon struct { environment string buildWatch bool + poll bool gc bool @@ -291,6 +292,7 @@ func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) { cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/") cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages") cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build") + cmd.Flags().BoolVar(&cc.poll, "poll", false, "use a poll based approach to watch for file system changes") cmd.Flags().Bool("templateMetrics", false, "display metrics about template executions") cmd.Flags().Bool("templateMetricsHints", false, "calculate some improvement hints when combined with --templateMetrics") diff --git a/commands/hugo.go b/commands/hugo.go index fddbe2b77fc..598a87cd48b 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -523,7 +523,7 @@ func (c *commandeer) build() error { c.logger.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs) c.logger.Println("Press Ctrl+C to stop") - watcher, err := c.newWatcher(watchDirs...) + watcher, err := c.newWatcher(c.h.poll, watchDirs...) checkErr(c.Logger, err) defer watcher.Close() @@ -820,7 +820,7 @@ func (c *commandeer) fullRebuild(changeType string) { } // newWatcher creates a new watcher to watch filesystem events. -func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { +func (c *commandeer) newWatcher(poll bool, dirList ...string) (*watcher.Batcher, error) { if runtime.GOOS == "darwin" { tweakLimit() } @@ -830,7 +830,7 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { return nil, err } - watcher, err := watcher.New(1 * time.Second) + watcher, err := watcher.New(1*time.Second, poll) if err != nil { return nil, err } @@ -859,7 +859,7 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { // Need to reload browser to show the error livereload.ForceRefresh() } - case err := <-watcher.Errors: + case err := <-watcher.Errors(): if err != nil { c.logger.Errorln("Error while watching:", err) } diff --git a/commands/server.go b/commands/server.go index 02db354ba56..5c8c778d40e 100644 --- a/commands/server.go +++ b/commands/server.go @@ -262,7 +262,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { for _, group := range watchGroups { jww.FEEDBACK.Printf("Watching for changes in %s\n", group) } - watcher, err := c.newWatcher(watchDirs...) + watcher, err := c.newWatcher(sc.poll, watchDirs...) if err != nil { return err } diff --git a/watcher/batcher.go b/watcher/batcher.go index 12c51940d7c..52e0dd42ee6 100644 --- a/watcher/batcher.go +++ b/watcher/batcher.go @@ -17,11 +17,12 @@ import ( "time" "github.com/fsnotify/fsnotify" + "github.com/gohugoio/hugo/watcher/filenotify" ) // Batcher batches file watch events in a given interval. type Batcher struct { - *fsnotify.Watcher + filenotify.FileWatcher interval time.Duration done chan struct{} @@ -29,11 +30,24 @@ type Batcher struct { } // New creates and starts a Batcher with the given time interval. -func New(interval time.Duration) (*Batcher, error) { - watcher, err := fsnotify.NewWatcher() +// It will fall back to a poll based watcher if native isn's supported. +// To always use pollng, set poll to true. +func New(interval time.Duration, poll bool) (*Batcher, error) { + var err error + var watcher filenotify.FileWatcher + + if poll { + watcher = filenotify.NewPollingWatcher() + } else { + watcher, err = filenotify.New() + } + + if err != nil { + return nil, err + } batcher := &Batcher{} - batcher.Watcher = watcher + batcher.FileWatcher = watcher batcher.interval = interval batcher.done = make(chan struct{}, 1) batcher.Events = make(chan []fsnotify.Event, 1) @@ -42,7 +56,7 @@ func New(interval time.Duration) (*Batcher, error) { go batcher.run() } - return batcher, err + return batcher, nil } func (b *Batcher) run() { @@ -51,7 +65,7 @@ func (b *Batcher) run() { OuterLoop: for { select { - case ev := <-b.Watcher.Events: + case ev := <-b.FileWatcher.Events(): evs = append(evs, ev) case <-tick: if len(evs) == 0 { @@ -69,5 +83,5 @@ OuterLoop: // Close stops the watching of the files. func (b *Batcher) Close() { b.done <- struct{}{} - b.Watcher.Close() + b.FileWatcher.Close() } diff --git a/watcher/filenotify/filenotify.go b/watcher/filenotify/filenotify.go new file mode 100644 index 00000000000..9d480bca15b --- /dev/null +++ b/watcher/filenotify/filenotify.go @@ -0,0 +1,43 @@ +// Package filenotify provides a mechanism for watching file(s) for changes. +// Generally leans on fsnotify, but provides a poll-based notifier which fsnotify does not support. +// These are wrapped up in a common interface so that either can be used interchangeably in your code. +// +// This package is adapted from https://github.com/moby/moby/tree/master/pkg/filenotify, Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package filenotify + +import "github.com/fsnotify/fsnotify" + +// FileWatcher is an interface for implementing file notification watchers +type FileWatcher interface { + Events() <-chan fsnotify.Event + Errors() <-chan error + Add(name string) error + Remove(name string) error + Close() error +} + +// New tries to use an fs-event watcher, and falls back to the poller if there is an error +func New() (FileWatcher, error) { + if watcher, err := NewEventWatcher(); err == nil { + return watcher, nil + } + return NewPollingWatcher(), nil +} + +// NewPollingWatcher returns a poll-based file watcher +func NewPollingWatcher() FileWatcher { + return &filePoller{ + events: make(chan fsnotify.Event), + errors: make(chan error), + } +} + +// NewEventWatcher returns an fs-event based file watcher +func NewEventWatcher() (FileWatcher, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + return &fsNotifyWatcher{watcher}, nil +} diff --git a/watcher/filenotify/fsnotify.go b/watcher/filenotify/fsnotify.go new file mode 100644 index 00000000000..57e39c09eb9 --- /dev/null +++ b/watcher/filenotify/fsnotify.go @@ -0,0 +1,20 @@ +// This package is adapted from https://github.com/moby/moby/tree/master/pkg/filenotify, Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package filenotify + +import "github.com/fsnotify/fsnotify" + +// fsNotifyWatcher wraps the fsnotify package to satisfy the FileNotifier interface +type fsNotifyWatcher struct { + *fsnotify.Watcher +} + +// Events returns the fsnotify event channel receiver +func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event { + return w.Watcher.Events +} + +// Errors returns the fsnotify error channel receiver +func (w *fsNotifyWatcher) Errors() <-chan error { + return w.Watcher.Errors +} diff --git a/watcher/filenotify/poller.go b/watcher/filenotify/poller.go new file mode 100644 index 00000000000..707606e8654 --- /dev/null +++ b/watcher/filenotify/poller.go @@ -0,0 +1,287 @@ +// This package is adapted from https://github.com/moby/moby/tree/master/pkg/filenotify, Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package filenotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +var ( + // errPollerClosed is returned when the poller is closed + errPollerClosed = errors.New("poller is closed") + // errNoSuchWatch is returned when trying to remove a watch that doesn't exist + errNoSuchWatch = errors.New("watch does not exist") +) + +// watchWaitTime is the time to wait between file poll loops +const watchWaitTime = 200 * time.Millisecond + +// filePoller is used to poll files for changes, especially in cases where fsnotify +// can't be run (e.g. when inotify handles are exhausted) +// filePoller satisfies the FileWatcher interface +type filePoller struct { + // watches is the list of files currently being polled, close the associated channel to stop the watch + watches map[string]chan struct{} + // events is the channel to listen to for watch events + events chan fsnotify.Event + // errors is the channel to listen to for watch errors + errors chan error + // mu locks the poller for modification + mu sync.Mutex + // closed is used to specify when the poller has already closed + closed bool +} + +// Add adds a filename to the list of watches +// once added the file is polled for changes in a separate goroutine +func (w *filePoller) Add(name string) error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.closed { + return errPollerClosed + } + + item, err := newItemToWatch(name) + if err != nil { + return err + } + + if w.watches == nil { + w.watches = make(map[string]chan struct{}) + } + if _, exists := w.watches[name]; exists { + return fmt.Errorf("watch exists") + } + chClose := make(chan struct{}) + w.watches[name] = chClose + + go w.watch(item, chClose) + return nil +} + +// Remove stops and removes watch with the specified name +func (w *filePoller) Remove(name string) error { + w.mu.Lock() + defer w.mu.Unlock() + return w.remove(name) +} + +func (w *filePoller) remove(name string) error { + if w.closed { + return errPollerClosed + } + + chClose, exists := w.watches[name] + if !exists { + return errNoSuchWatch + } + close(chClose) + delete(w.watches, name) + return nil +} + +// Events returns the event channel +// This is used for notifications on events about watched files +func (w *filePoller) Events() <-chan fsnotify.Event { + return w.events +} + +// Errors returns the errors channel +// This is used for notifications about errors on watched files +func (w *filePoller) Errors() <-chan error { + return w.errors +} + +// Close closes the poller +// All watches are stopped, removed, and the poller cannot be added to +func (w *filePoller) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.closed { + return nil + } + + for name := range w.watches { + w.remove(name) + } + w.closed = true + return nil +} + +// sendEvent publishes the specified event to the events channel +func (w *filePoller) sendEvent(e fsnotify.Event, chClose <-chan struct{}) error { + select { + case w.events <- e: + case <-chClose: + return fmt.Errorf("closed") + } + return nil +} + +// sendErr publishes the specified error to the errors channel +func (w *filePoller) sendErr(e error, chClose <-chan struct{}) error { + select { + case w.errors <- e: + case <-chClose: + return fmt.Errorf("closed") + } + return nil +} + +// itemToWatch may be a file or a dir. +type itemToWatch struct { + os.FileInfo + + // Full path to the filename. + filename string + + // Set if FileInfo is a dir. + entries map[string]os.FileInfo +} + +func newItemToWatch(filename string) (*itemToWatch, error) { + fi, err := os.Stat(filename) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + var lastDirEntries map[string]os.FileInfo + + // If fi is a dir, we watch the files inside that directory (not recursively). + // This matches the behaviour of fsnotity. + // TODO(bep) do we get event for the dir/too? + if fi != nil && fi.IsDir() { + lastDirEntries = make(map[string]os.FileInfo) + dirEntries, err := os.ReadDir(filename) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, dir := range dirEntries { + dfi, err := dir.Info() + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + lastDirEntries[dir.Name()] = dfi + } + } + + return &itemToWatch{FileInfo: fi, filename: filename, entries: lastDirEntries}, nil + +} + +func (item *itemToWatch) checkForChanges() (*itemToWatch, []fsnotify.Event, error) { + itemIn, err := newItemToWatch(item.filename) + if err != nil { + return nil, nil, err + } + + eventOp := func(fi1, fi2 os.FileInfo) fsnotify.Op { + if fi1 == nil && fi2 != nil { + return fsnotify.Create + } + if fi1 != nil && fi2 == nil { + return fsnotify.Remove + } + if fi1 == nil && fi2 == nil { + return 0 + } + if fi1.IsDir() || fi2.IsDir() { + return 0 + } + if fi1.Mode() != fi2.Mode() { + return fsnotify.Chmod + } + if fi1.ModTime() != fi2.ModTime() || fi1.Size() != fi2.Size() { + return fsnotify.Write + } + return 0 + } + + dirOp := eventOp(item.FileInfo, itemIn.FileInfo) + if dirOp != 0 { + evs := []fsnotify.Event{fsnotify.Event{Op: dirOp, Name: item.filename}} + return itemIn, evs, nil + } + + if item.FileInfo == nil || !item.IsDir() { + // Done. + return itemIn, nil, nil + } + + if item.entries == nil || len(item.entries) == 0 { + evs := []fsnotify.Event{fsnotify.Event{Op: fsnotify.Create, Name: item.filename}} + return itemIn, evs, nil + } + + if itemIn.entries == nil || len(itemIn.entries) == 0 { + evs := []fsnotify.Event{fsnotify.Event{Op: fsnotify.Remove, Name: item.filename}} + return itemIn, evs, nil + } + + leftIsIn := false + left, right := item.entries, itemIn.entries + if len(right) > len(left) { + left, right = right, left + leftIsIn = true + } + + var evs []fsnotify.Event + + for name, fi1 := range left { + fi2 := right[name] + fileft, firight := fi1, fi2 + if leftIsIn { + fileft, firight = firight, fileft + } + op := eventOp(fileft, firight) + if op != 0 { + evs = append(evs, fsnotify.Event{Op: op, Name: filepath.Join(item.filename, name)}) + } + + } + + return itemIn, evs, nil + +} + +// watch watches item for changes until done is closed. +func (w *filePoller) watch(item *itemToWatch, done chan struct{}) { + ticker := time.NewTicker(watchWaitTime) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + case <-done: + return + } + + itemIn, evs, err := item.checkForChanges() + if err != nil { + if err := w.sendErr(err, done); err != nil { + return + } + } + + item = itemIn + + for _, ev := range evs { + if err := w.sendEvent(ev, done); err != nil { + return + } + } + + } +} diff --git a/watcher/filenotify/poller_test.go b/watcher/filenotify/poller_test.go new file mode 100644 index 00000000000..eb616cee7d2 --- /dev/null +++ b/watcher/filenotify/poller_test.go @@ -0,0 +1,202 @@ +// This package is adapted from https://github.com/moby/moby/tree/master/pkg/filenotify, Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package filenotify + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + qt "github.com/frankban/quicktest" + "github.com/fsnotify/fsnotify" +) + +func TestPollerAddRemove(t *testing.T) { + w := NewPollingWatcher() + + if err := w.Add("no-such-file"); err == nil { + t.Fatal("should have gotten error when adding a non-existent file") + } + if err := w.Remove("no-such-file"); err == nil { + t.Fatal("should have gotten error when removing non-existent watch") + } + + f, err := ioutil.TempFile("", "asdf") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(f.Name()) + + if err := w.Add(f.Name()); err != nil { + t.Fatal(err) + } + + if err := w.Remove(f.Name()); err != nil { + t.Fatal(err) + } +} + +func TestPollerEvent(t *testing.T) { + c := qt.New(t) + + const ( + subdir1 = "subdir1" + subdir2 = "subdir2" + ) + + prepareTest := func(c *qt.C, poll bool) (string, FileWatcher) { + var w FileWatcher + if poll { + w = NewPollingWatcher() + } else { + var err error + w, err = NewEventWatcher() + c.Assert(err, qt.IsNil) + } + + dir, err := ioutil.TempDir("", fmt.Sprintf("test-poller-dir-%t", poll)) + c.Assert(err, qt.IsNil) + c.Assert(os.MkdirAll(filepath.Join(dir, subdir1), 0777), qt.IsNil) + c.Assert(os.MkdirAll(filepath.Join(dir, subdir2), 0777), qt.IsNil) + + for i := 0; i < 3; i++ { + c.Assert(ioutil.WriteFile(filepath.Join(dir, subdir1, fmt.Sprintf("file%d", i)), []byte("hello1"), 0600), qt.IsNil) + } + + for i := 0; i < 3; i++ { + c.Assert(ioutil.WriteFile(filepath.Join(dir, subdir2, fmt.Sprintf("file%d", i)), []byte("hello2"), 0600), qt.IsNil) + } + + c.Cleanup(func() { + w.Close() + os.RemoveAll(dir) + }) + return dir, w + + } + + for _, poll := range []bool{false, true} { + method := "fsnotify" + if poll { + method = "poll" + } + + c.Run(fmt.Sprintf("Watch dir, %s", method), func(c *qt.C) { + dir, w := prepareTest(c, poll) + subdir := filepath.Join(dir, subdir1) + c.Assert(w.Add(subdir), qt.IsNil) + + filename := filepath.Join(subdir, "file1") + + // Write to one file. + c.Assert(ioutil.WriteFile(filename, []byte("changed"), 0600), qt.IsNil) + assertFileMode(c, filename, 0600) + if !poll { + // fsnotify emits Chmod before Write + assertOneEvent(c, w, filename, fsnotify.Chmod) + } + assertOneEvent(c, w, filename, fsnotify.Write) + assertNoEvent(c, w) + + // Remove one file. + filename = filepath.Join(subdir, "file2") + c.Assert(os.Remove(filename), qt.IsNil) + assertOneEvent(c, w, filename, fsnotify.Remove) + assertNoEvent(c, w) + + // Add one file. + filename = filepath.Join(subdir, "file3") + c.Assert(ioutil.WriteFile(filename, []byte("new"), 0600), qt.IsNil) + assertOneEvent(c, w, filename, fsnotify.Create) + assertNoEvent(c, w) + + // Remove entire directory. + subdir = filepath.Join(dir, subdir2) + c.Assert(w.Add(subdir), qt.IsNil) + + c.Assert(os.RemoveAll(subdir), qt.IsNil) + + // This looks like a bug in fsnotify. There are + // 3 files in this directory, yet we get Remove events + // for one of them + the directory. + if !poll { + assertOneEvent(c, w, filepath.Join(subdir, "file2"), fsnotify.Remove) + } + assertOneEvent(c, w, subdir, fsnotify.Remove) + assertNoEvent(c, w) + + }) + } + +} + +func TestPollerClose(t *testing.T) { + w := NewPollingWatcher() + if err := w.Close(); err != nil { + t.Fatal(err) + } + // test double-close + if err := w.Close(); err != nil { + t.Fatal(err) + } + + f, err := ioutil.TempFile("", "asdf") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(f.Name()) + if err := w.Add(f.Name()); err == nil { + t.Fatal("should have gotten error adding watch for closed watcher") + } +} + +func assertFileMode(c *qt.C, fileName string, mode uint32) { + c.Helper() + f, err := os.Stat(fileName) + c.Assert(err, qt.IsNil) + c.Assert(f.Mode(), qt.Equals, os.FileMode(mode)) +} + +func assertOneEvent(c *qt.C, w FileWatcher, filename string, eType fsnotify.Op) { + c.Helper() + check := func() error { + for { + select { + case e := <-w.Events(): + if e.Op&eType != eType { + return fmt.Errorf("got wrong event type, expected %q: %v", eType, e.Op) + } else if filename != e.Name { + return fmt.Errorf("got wrong filename, expected %q: %v", filename, e.Name) + } + return nil + case e := <-w.Errors(): + return fmt.Errorf("got unexpected error waiting for events %v: %v", eType, e) + case <-time.After(watchWaitTime * 10): + return errors.New("timed out waiting for event") + } + } + } + c.Assert(check(), qt.IsNil) +} + +func assertNoEvent(c *qt.C, w FileWatcher) { + c.Helper() + check := func() error { + for { + select { + case e := <-w.Events(): + return fmt.Errorf("got event: %v", e) + case e := <-w.Errors(): + return fmt.Errorf("got unexpected error waiting for events: %v", e) + case <-time.After(watchWaitTime * 3): + // OK. + return nil + } + } + } + c.Assert(check(), qt.IsNil) +}