Implement BlackList (Alluxio#4)

bf8086 · Oct 2, 2019 · 50a9bd6 · 50a9bd6
1 parent 5ab45e3
commit 50a9bd6
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 7 deletions.
diff --git a/core/common/src/main/java/alluxio/conf/PropertyKey.java b/core/common/src/main/java/alluxio/conf/PropertyKey.java
@@ -1639,6 +1639,11 @@ public String toString() {
           .setDescription("How often the master schedules persistence jobs "
               + "for files written using ASYNC_THROUGH")
           .build();
+  public static final PropertyKey MASTER_PERSISTENCE_BLACKLIST =
+      new Builder(Name.MASTER_PERSISTENCE_BLACKLIST)
+          .setDefaultValue("")
+          .setDescription("Patterns to blacklist persist, comma separated, string match, no regex.")
+          .build();
   public static final PropertyKey MASTER_REPLICATION_CHECK_INTERVAL_MS =
       new Builder(Name.MASTER_REPLICATION_CHECK_INTERVAL_MS)
           .setDefaultValue("1min")
@@ -3922,6 +3927,8 @@ public static final class Name {
         "alluxio.master.persistence.max.interval";
     public static final String MASTER_PERSISTENCE_SCHEDULER_INTERVAL_MS =
         "alluxio.master.persistence.scheduler.interval";
+    public static final String MASTER_PERSISTENCE_BLACKLIST =
+        "alluxio.master.persistence.blacklist";
     public static final String MASTER_LOG_CONFIG_REPORT_HEARTBEAT_INTERVAL =
         "alluxio.master.log.config.report.heartbeat.interval";
     public static final String MASTER_PERIODIC_BLOCK_INTEGRITY_CHECK_REPAIR =

diff --git a/core/server/master/src/main/java/alluxio/master/file/DefaultFileSystemMaster.java b/core/server/master/src/main/java/alluxio/master/file/DefaultFileSystemMaster.java
@@ -2045,7 +2045,8 @@ private void renameInternal(RpcContext rpcContext, LockedInodePath srcInodePath,
 
     // Check options and determine if we should schedule async persist. This is helpful for compute
     // frameworks that use rename as a commit operation.
-    if (context.getPersist() && srcInode.isFile() && !srcInode.isPersisted()) {
+    if (context.getPersist() && srcInode.isFile() && !srcInode.isPersisted()
+        && shouldPersistPath(dstInodePath.toString())) {
       mInodeTree.updateInode(rpcContext, UpdateInodeEntry.newBuilder()
           .setId(srcInode.getId())
           .setPersistenceState(PersistenceState.TO_BE_PERSISTED.name())
@@ -2061,6 +2062,20 @@ private void renameInternal(RpcContext rpcContext, LockedInodePath srcInodePath,
     }
   }
 
+  private static final List<String> PERSISTENCE_BLACKLIST =
+      ServerConfiguration.getList(PropertyKey.MASTER_PERSISTENCE_BLACKLIST, ",");
+
+  private boolean shouldPersistPath(String path) {
+    for (String pattern : PERSISTENCE_BLACKLIST) {
+      if (path.contains(pattern)) {
+        LOG.debug("Not persisting path {} because it is in PERSISTENC_BLACKLIST {}",
+            path, PERSISTENCE_BLACKLIST);
+        return false;
+      }
+    }
+    return true;
+  }
+
   /**
    * Implements renaming.
    *
@@ -3151,14 +3166,16 @@ private void scheduleAsyncPersistenceInternal(LockedInodePath inodePath,
       throw new InvalidPathException(
           "Cannot persist an incomplete Alluxio file: " + inodePath.getUri());
     }
-    mInodeTree.updateInode(rpcContext, UpdateInodeEntry.newBuilder().setId(inode.getId())
+    if (shouldPersistPath(inodePath.toString())) {
+      mInodeTree.updateInode(rpcContext, UpdateInodeEntry.newBuilder().setId(inode.getId())
         .setPersistenceState(PersistenceState.TO_BE_PERSISTED.name()).build());
-    mPersistRequests.put(inode.getId(),
+      mPersistRequests.put(inode.getId(),
         new alluxio.time.ExponentialTimer(
-            ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS),
-            ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_INTERVAL_MS),
-            context.getPersistenceWaitTime(),
-            ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_TOTAL_WAIT_TIME_MS)));
+          ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS),
+          ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_INTERVAL_MS),
+          context.getPersistenceWaitTime(),
+          ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_TOTAL_WAIT_TIME_MS)));
+    }
   }
 
   /**