HBASE-24175 [Flakey Tests] TestSecureExportSnapshot FileNotFoundExcep…

…tion Addendum: add test to check for '/tmp' references.
apache · Apr 15, 2020 · 853a72b · 853a72b
1 parent 88c02c1
commit 853a72b
Show file tree

Hide file tree

Showing 2 changed files with 53 additions and 21 deletions.
diff --git a/...e-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java b/...e-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.hbase.snapshot;
 
+import static org.junit.Assert.assertFalse;
+import java.util.Iterator;
+import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -67,6 +70,20 @@ public static void setUpBeforeClass() throws Exception {
     TestExportSnapshot.setUpBaseConf(TEST_UTIL.getConfiguration());
     TEST_UTIL.startMiniCluster(3);
     TEST_UTIL.startMiniMapReduceCluster();
+    Configuration conf = TEST_UTIL.getConfiguration();
+    for (Iterator<Map.Entry<String, String>> i = conf.iterator(); i.hasNext();) {
+      Map.Entry<String, String> e = i.next();
+      if (e.getValue().contains("java.io.tmpdir")) {
+        continue;
+      }
+      if (e.getValue().contains("hadoop.tmp.dir")) {
+        continue;
+      }
+      if (e.getValue().contains("hbase.tmp.dir")) {
+        continue;
+      }
+      assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("tmp"));
+    }
   }
 
   @AfterClass

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -417,21 +417,6 @@ protected Path setupDataTestDir() {
     createSubDir(
       "mapreduce.cluster.local.dir",
       testPath, "mapred-local-dir");
-
-    // Frustrate yarn's attempts at writing /tmp.
-    String property = "yarn.node-labels.fs-store.root-dir";
-    createSubDir(property, testPath, property);
-    property = "yarn.nodemanager.log-dirs";
-    createSubDir(property, testPath, property);
-    property = "yarn.nodemanager.remote-app-log-dir";
-    createSubDir(property, testPath, property);
-    property = "yarn.timeline-service.entity-group-fs-store.active-dir";
-    createSubDir(property, testPath, property);
-    property = "yarn.timeline-service.entity-group-fs-store.done-dir";
-    createSubDir(property, testPath, property);
-    property = "yarn.nodemanager.remote-app-log-dir";
-    createSubDir(property, testPath, property);
-
     return testPath;
   }
 
@@ -650,16 +635,24 @@ public MiniDFSCluster startMiniDFSClusterForTestWAL(int namenodePort) throws IOE
     return dfsCluster;
   }
 
-  /** This is used before starting HDFS and map-reduce mini-clusters */
+  /** This is used before starting HDFS and map-reduce mini-clusters
+   * Run something like the below to check for the likes of '/tmp' references -- i.e.
+   * references outside of the test data dir -- in the conf.
+   *     Configuration conf = TEST_UTIL.getConfiguration();
+   *     for (Iterator<Map.Entry<String, String>> i = conf.iterator(); i.hasNext();) {
+   *       Map.Entry<String, String> e = i.next();
+   *       assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("/tmp"));
+   *     }
+   */
   private void createDirsAndSetProperties() throws IOException {
     setupClusterTestDir();
     conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
     System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
-    createDirAndSetProperty("cache_data", "test.cache.data");
-    createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
-    hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
-    createDirAndSetProperty("mapred_local", "mapreduce.cluster.local.dir");
-    createDirAndSetProperty("mapred_temp", "mapreduce.cluster.temp.dir");
+    createDirAndSetProperty("test.cache.data", "test.cache.data");
+    createDirAndSetProperty("hadoop.tmp.dir", "hadoop.tmp.dir");
+    hadoopLogDir = createDirAndSetProperty("hadoop.log.dir", "hadoop.log.dir");
+    createDirAndSetProperty("mapreduce.cluster.local.dir", "mapreduce.cluster.local.dir");
+    createDirAndSetProperty("mapreduce.cluster.temp.dir", "mapreduce.cluster.temp.dir");
     enableShortCircuit();
 
     Path root = getDataTestDirOnTestFS("hadoop");
@@ -671,6 +664,28 @@ private void createDirsAndSetProperties() throws IOException {
     conf.set("mapreduce.job.working.dir", new Path(root, "mapred-working-dir").toString());
     conf.set("yarn.app.mapreduce.am.staging-dir",
       new Path(root, "mapreduce-am-staging-root-dir").toString());
+
+    // Frustrate yarn's and hdfs's attempts at writing /tmp.
+    String property = "yarn.node-labels.fs-store.root-dir";
+    createDirAndSetProperty(property, property);
+    property = "yarn.nodemanager.log-dirs";
+    createDirAndSetProperty(property, property);
+    property = "yarn.nodemanager.remote-app-log-dir";
+    createDirAndSetProperty(property, property);
+    property = "yarn.timeline-service.entity-group-fs-store.active-dir";
+    createDirAndSetProperty(property, property);
+    property = "yarn.timeline-service.entity-group-fs-store.done-dir";
+    createDirAndSetProperty(property, property);
+    property = "yarn.nodemanager.remote-app-log-dir";
+    createDirAndSetProperty(property, property);
+    property = "dfs.journalnode.edits.dir";
+    createDirAndSetProperty(property, property);
+    property = "dfs.datanode.shared.file.descriptor.paths";
+    createDirAndSetProperty(property, property);
+    property = "nfs.dump.dir";
+    createDirAndSetProperty(property, property);
+    property = "java.io.tmpdir";
+    createDirAndSetProperty(property, property);
   }
 
   /**