Support watching independent files

README.md

@@ -4,6 +4,8 @@
 
 A directory watcher utility for JDK 8+ that aims to provide accurate and efficient recursive watching for Linux, macOS and Windows. In particular, this library provides a JNA-based `WatchService` for Mac OS X to replace the default polling-based JDK implementation.
 
+`directory-watcher` can watch both directories and independent-files.
+
 The core directory-watcher library is designed to have minimal dependencies; currently it only depends on `slf4j-api` (for internal logging, which can be disabled by passing a `NOPLogger` in the builder) and `jna` (for the macOS watcher implementation).
 
 ## Getting started
@@ -61,6 +63,7 @@ public class DirectoryWatchingUtility {
             case DELETE: /* file deleted */; break;
           }
         })
+        // .files(independentFilesToWatch)
         // .fileHashing(false) // defaults to true
         // .logger(logger) // defaults to LoggerFactory.getLogger(DirectoryWatcher.class)
         // .watchService(watchService) // defaults based on OS to either JVM WatchService or the JNA macOS WatchService

core/src/main/java/io/methvin/watcher/DirectoryWatcher.java

@@ -29,6 +29,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ForkJoinPool;
@@ -43,6 +44,7 @@ public class DirectoryWatcher {
    */
   public static final class Builder {
     private List<Path> paths = Collections.emptyList();
+    private List<Path> files = Collections.emptyList();
     private DirectoryChangeListener listener = (event -> {});
     private Logger logger = null;
     private FileHasher fileHasher = FileHasher.DEFAULT_FILE_HASHER;
@@ -58,6 +60,21 @@ public Builder paths(List<Path> paths) {
       return this;
     }
 
+    /**
+     * Set multiple files to watch.
+     *
+     * Note that the watch services interface does not have the power to watch independent
+     * files. Therefore, `directory-watcher` simulates watching these independent files
+     * by watching their parent directories non-recursively. This is the only performant
+     * way of supporting watching independent files.
+     *
+     * @param files Paths to files (they cannot be directories).
+     */
+    public Builder files(List<Path> files) {
+      this.files = files;
+      return this;
+    }
+
     /**
      * Set a single path to watch.
      */
@@ -117,7 +134,7 @@ public DirectoryWatcher build() throws IOException {
       if (logger == null) {
         staticLogger();
       }
-      return new DirectoryWatcher(paths, listener, watchService, fileHasher, logger);
+      return new DirectoryWatcher(paths, files, listener, watchService, fileHasher, logger);
     }
 
     private Builder osDefaultWatchService() throws IOException {
@@ -141,34 +158,45 @@ public static Builder builder() {
 
   private final WatchService watchService;
   private final List<Path> paths;
+  private final List<Path> files;
   private final boolean isMac;
   private final DirectoryChangeListener listener;
   private final Map<Path, HashCode> pathHashes;
   private final Map<WatchKey, Path> keyRoots;
+  private final Set<Path> watchingPathRoots;
+  private final Set<Path> nonRecursivePathRoots;
 
   // this is set to true/false depending on whether recursive watching is supported natively
   private Boolean fileTreeSupported = null;
   private FileHasher fileHasher;
 
   public DirectoryWatcher(
       List<Path> paths,
+      List<Path> files,
       DirectoryChangeListener listener,
       WatchService watchService,
       FileHasher fileHasher,
       Logger logger
   ) throws IOException {
     this.paths = paths;
+    this.files = files;
     this.listener = listener;
     this.watchService = watchService;
     this.isMac = watchService instanceof MacOSXListeningWatchService;
     this.pathHashes = PathUtils.createHashCodeMap(paths, fileHasher);
     this.keyRoots = PathUtils.createKeyRootsMap();
     this.fileHasher = fileHasher;
+    this.watchingPathRoots = PathUtils.createKeyRootsSet();
+    this.nonRecursivePathRoots = PathUtils.createKeyRootsSet();
     this.logger = logger;
 
+    /* Register all recursive paths after the non recursive paths to invalidate
+     * those non-recursive paths that are children of the recursive paths. */
     for (Path path : paths) {
       registerAll(path);
     }
+
+    registerRootsForFiles(files);
   }
 
   /**
@@ -216,8 +244,10 @@ public void watch() {
             throw new IllegalStateException(
                 "WatchService returned key [" + key + "] but it was not found in keyRoots!");
           }
+
           Path childPath = eventPath == null ? null : keyRoots.get(key).resolve(eventPath);
           logger.debug("{} [{}]", kind, childPath);
+
           // if directory is created, and watching recursively, then register it and its sub-directories
           if (kind == OVERFLOW) {
             listener.onEvent(new DirectoryChangeEvent(EventType.OVERFLOW, childPath, count));
@@ -226,8 +256,14 @@ public void watch() {
           } else if (kind == ENTRY_CREATE) {
             if (Files.isDirectory(childPath, NOFOLLOW_LINKS)) {
               if (!Boolean.TRUE.equals(fileTreeSupported)) {
-                registerAll(childPath);
+                  if (shouldIgnoreDirectoryEvent(childPath)) {
+                    logger.debug("Ignored {} [{}] because one of its parent is a non-recursive directory", kind, childPath);
+                    continue;
+                  } else {
+                    registerAll(childPath);
+                  }
               }
+
               // Our custom Mac service sends subdirectory changes but the Windows/Linux do not.
               // Walk the file tree to make sure we send create events for any files that were created.
               if (!isMac) {
@@ -281,6 +317,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
         logger.debug("WatchKey for [{}] no longer valid; removing.", key.watchable());
         // remove the key from the keyRoots
         keyRoots.remove(key);
+        watchingPathRoots.remove(key);
         // if there are no more keys left to watch, we can break out
         if (keyRoots.isEmpty()) {
           logger.debug("No more directories left to watch; terminating watcher.");
@@ -328,13 +365,73 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
   // Internal method to be used by registerAll
   private void register(Path directory, boolean useFileTreeModifier) throws IOException {
     logger.debug("Registering [{}].", directory);
-    Watchable watchable = isMac ? new WatchablePath(directory) : directory;
+    // If we don't use file tree, we don't register `directory` for recursive file watching
+    Watchable watchable = isMac ? new WatchablePath(directory, useFileTreeModifier) : directory;
     WatchEvent.Modifier[] modifiers = useFileTreeModifier
         ? new WatchEvent.Modifier[] {ExtendedWatchEventModifier.FILE_TREE}
         : new WatchEvent.Modifier[] {};
     WatchEvent.Kind<?>[] kinds = new WatchEvent.Kind<?>[] {ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY};
     WatchKey watchKey = watchable.register(watchService, kinds, modifiers);
     keyRoots.put(watchKey, directory);
+    watchingPathRoots.add(directory);
+  }
+
+  /**
+   * Register parents of independent files for non-recursive event watching.
+   *
+   * If any of the parents of the independent files have already been registered or
+   * are subsumed by a path that has been registered, they are skipped.
+   *
+   * @param files The independent files that we want to watch.
+   */
+  private void registerRootsForFiles(List<Path> files) throws IOException {
+    for (Path file: files) {
+      Path parentFile = file.getParent();
+     if (!watchingPathRoots.contains(parentFile)) {
+        if (watchingPathRoots.isEmpty()) {
+          nonRecursivePathRoots.add(parentFile);
+          register(parentFile, false);
+        } else {
+          for (Path watchRoot : watchingPathRoots) {
+            if (parentFile.startsWith(watchRoot)) {
+              logger.debug("Parent file " + parentFile.toString() + " is contained in " + watchRoot);
+            } else {
+              nonRecursivePathRoots.add(parentFile);
+              register(parentFile, false);
+            }
+          }
+        }
+      } else {
+        logger.debug("Parent file " + parentFile.toString() + " already is being watched");
+      }
+    }
+  }
+
+  /**
+   * Checks whether a directory path associated with an event should be processed or not.
+   * Note that the argument path needs to represent a directory.
+   *
+   * A more efficient implementation could be done by checking directly that the parent
+   * of a path is a parent of a non-recursive directory. However, this implementation is
+   * not viable as the order of the event processing is not ensured.
+   *
+   * For example, take a non-recursive directory `foo`. If a user creates `foo/bar/foo.txt`,
+   * then there's no guarantee that the creation event of the directory `foo/bar` will be
+   * processed before the creation event of `foo/bar/foo.txt`.
+   *
+   * @param directory The direcotry path associated with an event of any kind.
+   * @return Whether directory watcher should process the event or not.
+   */
+  private boolean shouldIgnoreDirectoryEvent(Path directory) {
+    boolean ignoreEvent = false;
+    Path parentPath = directory.getParent();
+    for (Path nonRecursivePath : nonRecursivePathRoots) {
+      if (parentPath.equals(nonRecursivePath) && directory.startsWith(nonRecursivePath)) {
+        ignoreEvent = true;
+        break; // We're done here, no need to check the rest of the paths
+      }
+    }
+    return ignoreEvent;
   }
 
   private void notifyCreateEvent(Path path, int count) throws IOException {
@@ -355,5 +452,4 @@ private void notifyCreateEvent(Path path, int count) throws IOException {
       listener.onEvent(new DirectoryChangeEvent(EventType.CREATE, path, count));
     }
   }
-
 }

core/src/main/java/io/methvin/watcher/PathUtils.java

@@ -46,6 +46,10 @@ public static Map<WatchKey, Path> createKeyRootsMap() {
     return new ConcurrentHashMap<>();
   }
 
+  public static Set<Path> createKeyRootsSet() {
+    return ConcurrentHashMap.newKeySet();
+  }
+
   public static Map<Path, HashCode> createHashCodeMap(Path file, FileHasher fileHasher) {
     return createHashCodeMap(Collections.singletonList(file), fileHasher);
   }
@@ -65,6 +69,22 @@ public static Map<Path, HashCode> createHashCodeMap(List<Path> files, FileHasher
     return lastModifiedMap;
   }
 
+  public static Set<Path> nonRecursiveListFiles(Path file) {
+    Set<Path> files = new HashSet<Path>();
+    files.add(file);
+    if (file.toFile().isDirectory()) {
+      File[] filesInDirectory = file.toFile().listFiles();
+      if (filesInDirectory != null) {
+        for (File child : filesInDirectory) {
+          if (!child.isDirectory()) {
+            files.add(child.toPath());
+          }
+        }
+      }
+    }
+    return files;
+  }
+
   public static Set<Path> recursiveListFiles(Path file) {
     Set<Path> files = new HashSet<Path>();
     files.add(file);

core/src/main/java/io/methvin/watchservice/MacOSXListeningWatchService.java

@@ -104,7 +104,7 @@ public AbstractWatchKey register(WatchablePath watchable, Iterable<? extends Wat
     final CFArrayRef pathsToWatch = CarbonAPI.INSTANCE.CFArrayCreate(null, values, CFIndex.valueOf(1), null);
     final long kFSEventStreamEventIdSinceNow = -1; //  this is 0xFFFFFFFFFFFFFFFF
     final int kFSEventStreamCreateFlagNoDefer = 0x00000002;
-    final CarbonAPI.FSEventStreamCallback callback = new MacOSXListeningCallback(watchKey, fileHasher, hashCodeMap);
+    final CarbonAPI.FSEventStreamCallback callback = new MacOSXListeningCallback(watchKey, fileHasher, hashCodeMap, watchable);
     callbackList.add(callback);
     final FSEventStreamRef stream = CarbonAPI.INSTANCE.FSEventStreamCreate(
       Pointer.NULL,
@@ -166,11 +166,13 @@ private static class MacOSXListeningCallback implements CarbonAPI.FSEventStreamC
     private final MacOSXWatchKey watchKey;
     private final Map<Path, HashCode> hashCodeMap;
     private final FileHasher fileHasher;
+    private final WatchablePath watchable;
 
-    private MacOSXListeningCallback(MacOSXWatchKey watchKey, FileHasher fileHasher, Map<Path, HashCode> hashCodeMap) {
+    private MacOSXListeningCallback(MacOSXWatchKey watchKey, FileHasher fileHasher, Map<Path, HashCode> hashCodeMap, WatchablePath watchable) {
       this.watchKey = watchKey;
       this.hashCodeMap = hashCodeMap;
       this.fileHasher = fileHasher;
+      this.watchable = watchable;
     }
 
     @Override
@@ -179,8 +181,18 @@ public void invoke(FSEventStreamRef streamRef, Pointer clientCallBackInfo, Nativ
       final int length = numEvents.intValue();
 
       for (String folderName : eventPaths.getStringArray(0, length)) {
-        final Set<Path> filesOnDisk = PathUtils.recursiveListFiles(new File(folderName).toPath());
-        //
+        final Path folderPath = new File(folderName).toPath();
+        final Set<Path> filesOnDisk = new HashSet<Path>();
+        if (this.watchable.isRecursive()) {
+          filesOnDisk.addAll(PathUtils.recursiveListFiles(new File(folderName).toPath()));
+        } else {
+          // Ignore directory-level events whose path is a subpath of a non-recursive watchable
+          Path watchPath = this.watchable.getFile();
+          if (!folderPath.getParent().startsWith(watchPath)) {
+            filesOnDisk.addAll(PathUtils.nonRecursiveListFiles(new File(folderName).toPath()));
+          }
+        }
+
         // We collect and process all actions for each category of created, modified and deleted as it appears a first thread
         // can start while a second thread can get through faster. If we do the collection for each category in a second
         // thread can get to the processing of modifications before the first thread is finished processing creates.
@@ -190,7 +202,6 @@ public void invoke(FSEventStreamRef streamRef, Pointer clientCallBackInfo, Nativ
         // together the last modified time is not granular enough to be seen as a modification. This likely mitigates
         // the issue I originally saw where the ordering was incorrect but I will leave the collection and processing
         // of each category together.
-        //
 
         for (Path file : findCreatedFiles(filesOnDisk)) {
           if (watchKey.isReportCreateEvents()) {

core/src/main/java/io/methvin/watchservice/WatchablePath.java

@@ -25,18 +25,24 @@
 public class WatchablePath implements Watchable {
 
   private final Path file;
+  private final boolean isRecursive;
 
-  public WatchablePath(Path file) {
+  public WatchablePath(Path file, boolean isRecursive) {
     if (file == null) {
       throw new NullPointerException("file must not be null");
     }
     this.file = file;
+    this.isRecursive = isRecursive;
   }
 
   public Path getFile() {
     return file;
   }
 
+  public boolean isRecursive() {
+    return this.isRecursive;
+  }
+
   @Override
   public WatchKey register(WatchService watcher,
     WatchEvent.Kind<?>[] events,